diff options
author | Ilnaz Nizametdinov <i.nizametdinov@gmail.com> | 2022-02-10 16:48:01 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:01 +0300 |
commit | 17644d1207cae94abb87ae27d63dbf40e477cda4 (patch) | |
tree | b222e5ac2e2e98872661c51ccceee5da0d291e13 | |
parent | 86c3bbf7131fc3929bdb31e29fdf016038d68594 (diff) | |
download | ydb-17644d1207cae94abb87ae27d63dbf40e477cda4.tar.gz |
Restoring authorship annotation for Ilnaz Nizametdinov <i.nizametdinov@gmail.com>. Commit 2 of 2.
871 files changed, 76721 insertions, 76721 deletions
diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h index 7b75018619d..ba54baaa835 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h @@ -180,22 +180,22 @@ namespace Aws */ Aws::String caPath; /** - * Same as caPath, but useful if you have users going through a proxy. - * Used to set CURLOPT_PROXY_CAPATH in libcurl. - */ - Aws::String proxyCaPath; - /** + * Same as caPath, but useful if you have users going through a proxy. + * Used to set CURLOPT_PROXY_CAPATH in libcurl. + */ + Aws::String proxyCaPath; + /** * If you certificate file is different from the default, you can tell clients that * aren't using the default trust store where to find your ca file. * If you are on windows or apple, you likely don't want this. */ Aws::String caFile; /** - * Same as caFile, but useful if you have users going through a proxy. - * Used to set CURLOPT_PROXY_CAINFO in libcurl. - */ - Aws::String proxyCaFile; - /** + * Same as caFile, but useful if you have users going through a proxy. + * Used to set CURLOPT_PROXY_CAINFO in libcurl. + */ + Aws::String proxyCaFile; + /** * Rate Limiter implementation for outgoing bandwidth. Default is wide-open. */ std::shared_ptr<Aws::Utils::RateLimits::RateLimiterInterface> writeRateLimiter; diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/http/curl/CurlHttpClient.h b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/http/curl/CurlHttpClient.h index c3051b81526..31f96befd56 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/http/curl/CurlHttpClient.h +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/http/curl/CurlHttpClient.h @@ -63,8 +63,8 @@ private: bool m_verifySSL; Aws::String m_caPath; Aws::String m_caFile; - Aws::String m_proxyCaPath; - Aws::String m_proxyCaFile; + Aws::String m_proxyCaPath; + Aws::String m_proxyCaFile; bool m_disableExpectHeader; bool m_allowRedirects; static std::atomic<bool> isInit; diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/Outcome.h b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/Outcome.h index 6683e3d8601..70e9e4be262 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/Outcome.h +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/Outcome.h @@ -26,19 +26,19 @@ namespace Aws { public: - Outcome() : result(), error(), success(false) + Outcome() : result(), error(), success(false) { } - Outcome(const R& r) : result(r), error(), success(true) + Outcome(const R& r) : result(r), error(), success(true) { } - Outcome(const E& e) : result(), error(e), success(false) + Outcome(const E& e) : result(), error(e), success(false) { } - Outcome(R&& r) : result(std::forward<R>(r)), error(), success(true) + Outcome(R&& r) : result(std::forward<R>(r)), error(), success(true) { } - Outcome(E&& e) : result(), error(std::forward<E>(e)), success(false) + Outcome(E&& e) : result(), error(std::forward<E>(e)), success(false) { } Outcome(const Outcome& o) : diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp index 3db94925b8e..2fb9cc9643f 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp @@ -444,7 +444,7 @@ CurlHttpClient::CurlHttpClient(const ClientConfiguration& clientConfig) : m_proxySSLKeyPath(clientConfig.proxySSLKeyPath), m_proxySSLKeyType(clientConfig.proxySSLKeyType), m_proxyKeyPasswd(clientConfig.proxySSLKeyPassword), m_proxyPort(clientConfig.proxyPort), m_verifySSL(clientConfig.verifySSL), m_caPath(clientConfig.caPath), - m_caFile(clientConfig.caFile), m_proxyCaPath(clientConfig.proxyCaPath), m_proxyCaFile(clientConfig.proxyCaFile), + m_caFile(clientConfig.caFile), m_proxyCaPath(clientConfig.proxyCaPath), m_proxyCaFile(clientConfig.proxyCaFile), m_disableExpectHeader(clientConfig.disableExpectHeader) { if (clientConfig.followRedirects == FollowRedirectsPolicy::NEVER || @@ -582,14 +582,14 @@ std::shared_ptr<HttpResponse> CurlHttpClient::MakeRequest(const std::shared_ptr< ss << m_proxyScheme << "://" << m_proxyHost; curl_easy_setopt(connectionHandle, CURLOPT_PROXY, ss.str().c_str()); curl_easy_setopt(connectionHandle, CURLOPT_PROXYPORT, (long) m_proxyPort); - if(!m_proxyCaPath.empty()) - { - curl_easy_setopt(connectionHandle, CURLOPT_PROXY_CAPATH, m_proxyCaPath.c_str()); - } - if(!m_proxyCaFile.empty()) - { - curl_easy_setopt(connectionHandle, CURLOPT_PROXY_CAINFO, m_proxyCaFile.c_str()); - } + if(!m_proxyCaPath.empty()) + { + curl_easy_setopt(connectionHandle, CURLOPT_PROXY_CAPATH, m_proxyCaPath.c_str()); + } + if(!m_proxyCaFile.empty()) + { + curl_easy_setopt(connectionHandle, CURLOPT_PROXY_CAINFO, m_proxyCaFile.c_str()); + } if (!m_proxyUserName.empty() || !m_proxyPassword.empty()) { curl_easy_setopt(connectionHandle, CURLOPT_PROXYUSERNAME, m_proxyUserName.c_str()); diff --git a/library/cpp/actors/dnscachelib/dnscache.cpp b/library/cpp/actors/dnscachelib/dnscache.cpp index 60c8f9452c1..649339ddb22 100644 --- a/library/cpp/actors/dnscachelib/dnscache.cpp +++ b/library/cpp/actors/dnscachelib/dnscache.cpp @@ -76,7 +76,7 @@ TIpHost TDnsCache::Get(const TString& hostname) { const THost& addr = Resolve(hostname, AF_INET); - TGuard<TMutex> lock(CacheMtx); + TGuard<TMutex> lock(CacheMtx); if (addr.AddrsV4.empty()) { return TIpHost(-1); } @@ -91,7 +91,7 @@ NAddr::IRemoteAddrPtr TDnsCache::GetAddr( if (family != AF_INET && AllowIpV6) { const THost& addr = Resolve(hostname, AF_INET6, cacheOnly); - TGuard<TMutex> lock(CacheMtx); + TGuard<TMutex> lock(CacheMtx); if (!addr.AddrsV6.empty()) { struct sockaddr_in6 sin6; Zero(sin6); @@ -106,7 +106,7 @@ NAddr::IRemoteAddrPtr TDnsCache::GetAddr( if (family != AF_INET6 && AllowIpV4) { const THost& addr = Resolve(hostname, AF_INET, cacheOnly); - TGuard<TMutex> lock(CacheMtx); + TGuard<TMutex> lock(CacheMtx); if (!addr.AddrsV4.empty()) { return MakeHolder<NAddr::TIPv4Addr>(TIpAddress(addr.AddrsV4.front(), port)); } @@ -121,8 +121,8 @@ void TDnsCache::GetAllAddresses( TVector<NAddr::IRemoteAddrPtr>& addrs) { if (AllowIpV4) { const THost& addr4 = Resolve(hostname, AF_INET); - - TGuard<TMutex> lock(CacheMtx); + + TGuard<TMutex> lock(CacheMtx); for (size_t i = 0; i < addr4.AddrsV4.size(); i++) { addrs.push_back(MakeHolder<NAddr::TIPv4Addr>(TIpAddress(addr4.AddrsV4[i], 0))); } @@ -130,12 +130,12 @@ void TDnsCache::GetAllAddresses( if (AllowIpV6) { const THost& addr6 = Resolve(hostname, AF_INET6); - + struct sockaddr_in6 sin6; Zero(sin6); sin6.sin6_family = AF_INET6; - - TGuard<TMutex> lock(CacheMtx); + + TGuard<TMutex> lock(CacheMtx); for (size_t i = 0; i < addr6.AddrsV6.size(); i++) { sin6.sin6_addr = addr6.AddrsV6[i]; diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index d8ac88f3ae5..6fa25b99656 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -377,7 +377,7 @@ namespace NActors { TlsActivationContext = &ctx; recipientActor->Receive(ev, ctx); TlsActivationContext = prevTlsActivationContext; - // we expect the logger to never die in tests + // we expect the logger to never die in tests } } } else { diff --git a/library/cpp/monlib/service/pages/templates.cpp b/library/cpp/monlib/service/pages/templates.cpp index 5c2017bf5ed..ece12bea710 100644 --- a/library/cpp/monlib/service/pages/templates.cpp +++ b/library/cpp/monlib/service/pages/templates.cpp @@ -28,8 +28,8 @@ namespace NMonitoring { extern const char ListTag[] = "li"; extern const char UListTag[] = "ul"; extern const char OListTag[] = "ol"; - extern const char DListTag[] = "dl"; - extern const char DTermTag[] = "dt"; - extern const char DDescTag[] = "dd"; + extern const char DListTag[] = "dl"; + extern const char DTermTag[] = "dt"; + extern const char DDescTag[] = "dd"; } diff --git a/library/cpp/monlib/service/pages/templates.h b/library/cpp/monlib/service/pages/templates.h index fa6c986dcaf..b4656f059fa 100644 --- a/library/cpp/monlib/service/pages/templates.h +++ b/library/cpp/monlib/service/pages/templates.h @@ -78,13 +78,13 @@ #define OL() TAG(TOLIST) #define OL_CLASS(cls) TAG_CLASS(TOLIST, cls) -#define DL() TAG(DLIST) -#define DL_CLASS(cls) TAG_CLASS(DLIST, cls) -#define DT() TAG(DTERM) -#define DT_CLASS(cls) TAG_CLASS(DTERM, cls) -#define DD() TAG(DDESC) -#define DD_CLASS(cls) TAG_CLASS(DDESC, cls) - +#define DL() TAG(DLIST) +#define DL_CLASS(cls) TAG_CLASS(DLIST, cls) +#define DT() TAG(DTERM) +#define DT_CLASS(cls) TAG_CLASS(DTERM, cls) +#define DD() TAG(DDESC) +#define DD_CLASS(cls) TAG_CLASS(DDESC, cls) + #define CAPTION() TAG(TCaption) #define CAPTION_CLASS(cls) CAPTION_CLASS(TCaption, cls) @@ -231,9 +231,9 @@ namespace NMonitoring { extern const char ListTag[3]; extern const char UListTag[3]; extern const char OListTag[3]; - extern const char DListTag[3]; - extern const char DTermTag[3]; - extern const char DDescTag[3]; + extern const char DListTag[3]; + extern const char DTermTag[3]; + extern const char DDescTag[3]; typedef TTag<HtmlTag> THtml; typedef TTag<HeadTag> THead; @@ -262,7 +262,7 @@ namespace NMonitoring { typedef TTag<ListTag> TLIST; typedef TTag<UListTag> TULIST; typedef TTag<OListTag> TOLIST; - typedef TTag<DListTag> DLIST; - typedef TTag<DTermTag> DTERM; - typedef TTag<DDescTag> DDESC; + typedef TTag<DListTag> DLIST; + typedef TTag<DTermTag> DTERM; + typedef TTag<DDescTag> DDESC; } diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp index 62c09b33417..f9e517fc424 100644 --- a/ydb/core/base/appdata.cpp +++ b/ydb/core/base/appdata.cpp @@ -46,7 +46,7 @@ TAppData::TDefaultTabletTypes::TDefaultTabletTypes() , ColumnShard(TTabletTypes::COLUMNSHARD) , TestShard(TTabletTypes::TestShard) , SequenceShard(TTabletTypes::SequenceShard) - , ReplicationController(TTabletTypes::ReplicationController) + , ReplicationController(TTabletTypes::ReplicationController) { } diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index 80a0c66bc7f..c666f7468c0 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -3,7 +3,7 @@ #include "defs.h" #include "channel_profiles.h" #include "domain.h" -#include "feature_flags.h" +#include "feature_flags.h" #include "nameservice.h" #include "tablet_types.h" #include "resource_profile.h" @@ -60,7 +60,7 @@ namespace NSQS { namespace NPQ { class IPersQueueMirrorReaderFactory; } - + class TFormatFactory; struct TAppData { @@ -84,7 +84,7 @@ struct TAppData { IActor*(*FolderServiceFactory)(const NKikimrProto::NFolderService::TFolderServiceConfig&); const NPQ::IPersQueueMirrorReaderFactory* PersQueueMirrorReaderFactory = nullptr; - NYdb::TDriver* YdbDriver = nullptr; + NYdb::TDriver* YdbDriver = nullptr; const NPDisk::IIoContextFactory* IoContextFactory = nullptr; struct TDefaultTabletTypes { @@ -102,7 +102,7 @@ struct TAppData { TTabletTypes::EType ColumnShard; TTabletTypes::EType TestShard; TTabletTypes::EType SequenceShard; - TTabletTypes::EType ReplicationController; + TTabletTypes::EType ReplicationController; TDefaultTabletTypes(); }; @@ -141,10 +141,10 @@ struct TAppData { NKikimrProto::TAuthConfig AuthConfig; NKikimrProto::TKeyConfig KeyConfig; NKikimrProto::TKeyConfig PDiskKeyConfig; - TFeatureFlags FeatureFlags; + TFeatureFlags FeatureFlags; NKikimrConfig::THiveConfig HiveConfig; NKikimrConfig::TDataShardConfig DataShardConfig; - NKikimrConfig::TMeteringConfig MeteringConfig; + NKikimrConfig::TMeteringConfig MeteringConfig; NKikimrConfig::TCompactionConfig CompactionConfig; bool EnforceUserTokenRequirement = false; bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request diff --git a/ydb/core/base/cputime.h b/ydb/core/base/cputime.h index 5e89204e30e..6661a728faf 100644 --- a/ydb/core/base/cputime.h +++ b/ydb/core/base/cputime.h @@ -17,13 +17,13 @@ class TCpuTimer { } public: TCpuTimer() - : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) + : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) , StartTime(GetCpuTime()) , Counter(nullptr) {} TCpuTimer(TDuration& counter) - : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) + : UseClockGettime(AppData()->FeatureFlags.GetEnableClockGettimeForUserCpuAccounting()) , StartTime(GetCpuTime()) , Counter(&counter) {} diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 55ecc135ee7..f5fedfe19b2 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -36,7 +36,7 @@ struct TKikimrEvents : TEvents { ES_TX_DATASHARD, ES_DEPRECATED_4114, ES_TX_USERPROXY, // user proxy interface - ES_SCHEME_CACHE, + ES_SCHEME_CACHE, ES_TX_PROXY_REQ, ES_TABLET_PIPE, ES_DEPRECATED_4118, @@ -104,11 +104,11 @@ struct TKikimrEvents : TEvents { ES_USER_ACCOUNT_SERVICE, ES_PQ_PROXY_NEW, ES_GRPC_STREAMING, - ES_SCHEME_BOARD, - ES_FLAT_TX_SCHEMESHARD_PROTECTED, - ES_GRPC_REQUEST_PROXY, - ES_EXPORT_SERVICE, - ES_TX_ALLOCATOR_CLIENT, + ES_SCHEME_BOARD, + ES_FLAT_TX_SCHEMESHARD_PROTECTED, + ES_GRPC_REQUEST_PROXY, + ES_EXPORT_SERVICE, + ES_TX_ALLOCATOR_CLIENT, ES_PQ_CLUSTER_TRACKER, ES_NET_CLASSIFIER, ES_SYSTEM_VIEW, @@ -116,8 +116,8 @@ struct TKikimrEvents : TEvents { ES_SERVICE_ACCOUNT_SERVICE, ES_INDEX_BUILD, ES_BLOCKSTORE_PRIVATE, - ES_YT_WRAPPER, - ES_S3_WRAPPER, + ES_YT_WRAPPER, + ES_S3_WRAPPER, ES_FILESTORE, ES_FILESTORE_PRIVATE, ES_YDB_METERING, @@ -125,7 +125,7 @@ struct TKikimrEvents : TEvents { ES_TX_OLAPSHARD, ES_TX_COLUMNSHARD, ES_CROSSREF, - ES_SCHEME_BOARD_MON, + ES_SCHEME_BOARD_MON, ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY, ES_BLOB_CACHE, ES_LONG_TX_SERVICE, @@ -135,16 +135,16 @@ struct TKikimrEvents : TEvents { ES_HEALTH_CHECK, ES_DQ = NYql::NDq::TDqEvents::ES_DQ_COMPUTE, // 4212 ES_YQ, // 4213 - ES_CHANGE_EXCHANGE, + ES_CHANGE_EXCHANGE, ES_DATABASE_SERVICE, //4215 ES_SEQUENCESHARD, // 4216 ES_SEQUENCEPROXY, // 4217 ES_CLOUD_STORAGE, ES_CLOUD_STORAGE_PRIVATE, ES_FOLDER_SERVICE_ADAPTER, - ES_PQ_PARTITION_WRITER, - ES_YDB_PROXY, - ES_REPLICATION_CONTROLLER, + ES_PQ_PARTITION_WRITER, + ES_YDB_PROXY, + ES_REPLICATION_CONTROLLER, }; }; diff --git a/ydb/core/base/feature_flags.h b/ydb/core/base/feature_flags.h index 3e7a2f032c5..75b7158fa71 100644 --- a/ydb/core/base/feature_flags.h +++ b/ydb/core/base/feature_flags.h @@ -1,50 +1,50 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/protos/config.pb.h> - -namespace NKikimr { - -class TFeatureFlags: public NKikimrConfig::TFeatureFlags { - using TBase = NKikimrConfig::TFeatureFlags; - -public: - using TBase::TBase; - using TBase::operator=; - - inline std::optional<bool> GetEnableMvcc() const { - switch (TBase::GetEnableMvcc()) { - case NKikimrConfig::TFeatureFlags::UNSET: - return std::nullopt; - case NKikimrConfig::TFeatureFlags::VALUE_TRUE: - return true; - case NKikimrConfig::TFeatureFlags::VALUE_FALSE: - return false; - } - } - - inline void SetEnableMvccSnapshotReadsForTest(bool value) { - SetEnableMvccSnapshotReads(value); - } - - inline void SetEnableBackgroundCompactionForTest(bool value) { - SetEnableBackgroundCompaction(value); - } - - inline void SetEnableBackgroundCompactionServerlessForTest(bool value) { - SetEnableBackgroundCompactionServerless(value); - } - - inline void SetEnableOlapSchemaOperationsForTest(bool value) { - SetEnableOlapSchemaOperations(value); - } - - inline void SetEnableMvccForTest(bool value) { - SetEnableMvcc(value - ? NKikimrConfig::TFeatureFlags::VALUE_TRUE - : NKikimrConfig::TFeatureFlags::VALUE_FALSE); - } -}; - -} // NKikimr + +namespace NKikimr { + +class TFeatureFlags: public NKikimrConfig::TFeatureFlags { + using TBase = NKikimrConfig::TFeatureFlags; + +public: + using TBase::TBase; + using TBase::operator=; + + inline std::optional<bool> GetEnableMvcc() const { + switch (TBase::GetEnableMvcc()) { + case NKikimrConfig::TFeatureFlags::UNSET: + return std::nullopt; + case NKikimrConfig::TFeatureFlags::VALUE_TRUE: + return true; + case NKikimrConfig::TFeatureFlags::VALUE_FALSE: + return false; + } + } + + inline void SetEnableMvccSnapshotReadsForTest(bool value) { + SetEnableMvccSnapshotReads(value); + } + + inline void SetEnableBackgroundCompactionForTest(bool value) { + SetEnableBackgroundCompaction(value); + } + + inline void SetEnableBackgroundCompactionServerlessForTest(bool value) { + SetEnableBackgroundCompactionServerless(value); + } + + inline void SetEnableOlapSchemaOperationsForTest(bool value) { + SetEnableOlapSchemaOperations(value); + } + + inline void SetEnableMvccForTest(bool value) { + SetEnableMvcc(value + ? NKikimrConfig::TFeatureFlags::VALUE_TRUE + : NKikimrConfig::TFeatureFlags::VALUE_FALSE); + } +}; + +} // NKikimr diff --git a/ydb/core/base/localdb.cpp b/ydb/core/base/localdb.cpp index 8949fe3979d..d07ee26fc1f 100644 --- a/ydb/core/base/localdb.cpp +++ b/ydb/core/base/localdb.cpp @@ -1,5 +1,5 @@ #include "localdb.h" -#include "compile_time_flags.h" +#include "compile_time_flags.h" #include <ydb/core/protos/resource_broker.pb.h> diff --git a/ydb/core/base/path.cpp b/ydb/core/base/path.cpp index 91b78f24727..0cfb16f99dc 100644 --- a/ydb/core/base/path.cpp +++ b/ydb/core/base/path.cpp @@ -1,10 +1,10 @@ #include "path.h" -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/string/printf.h> namespace NKikimr { - + TVector<TString> SplitPath(TString path) { TVector<TString> res; if (path.empty()) @@ -161,47 +161,47 @@ TStringBuf ExtractBase(const TString &path) noexcept { return parent.RAfter('/'); } -bool TrySplitPathByDb(const TString& path, const TString& database, - std::pair<TString, TString>& result, TString& error) -{ - auto makeWrongDbError = [&]() { - return TStringBuilder() << "Table path not in database, path: " << path << ", database: " << database; - }; - - auto pathParts = SplitPath(path); - auto databaseParts = SplitPath(database); - - if (pathParts.size() <= databaseParts.size()) { - error = makeWrongDbError(); - return false; - } - - if (databaseParts.empty()) { - if (pathParts.size() < 2) { - error = TStringBuilder() << "Bad table path: " << path; - return false; - } - - result = std::make_pair( - CombinePath(pathParts.begin(), pathParts.begin() + 1), - CombinePath(pathParts.begin() + 1, pathParts.end(), false)); - - return true; - } - - for (ui32 i = 0; i < databaseParts.size(); ++i) { - if (pathParts[i] != databaseParts[i]) { - error = makeWrongDbError(); - return false; - } - } - - result = std::make_pair( - CombinePath(databaseParts.begin(), databaseParts.end()), - CombinePath(pathParts.begin() + databaseParts.size(), pathParts.end(), false) - ); - - return true; +bool TrySplitPathByDb(const TString& path, const TString& database, + std::pair<TString, TString>& result, TString& error) +{ + auto makeWrongDbError = [&]() { + return TStringBuilder() << "Table path not in database, path: " << path << ", database: " << database; + }; + + auto pathParts = SplitPath(path); + auto databaseParts = SplitPath(database); + + if (pathParts.size() <= databaseParts.size()) { + error = makeWrongDbError(); + return false; + } + + if (databaseParts.empty()) { + if (pathParts.size() < 2) { + error = TStringBuilder() << "Bad table path: " << path; + return false; + } + + result = std::make_pair( + CombinePath(pathParts.begin(), pathParts.begin() + 1), + CombinePath(pathParts.begin() + 1, pathParts.end(), false)); + + return true; + } + + for (ui32 i = 0; i < databaseParts.size(); ++i) { + if (pathParts[i] != databaseParts[i]) { + error = makeWrongDbError(); + return false; + } + } + + result = std::make_pair( + CombinePath(databaseParts.begin(), databaseParts.end()), + CombinePath(pathParts.begin() + databaseParts.size(), pathParts.end(), false) + ); + + return true; +} + } - -} diff --git a/ydb/core/base/path.h b/ydb/core/base/path.h index 11d77f033d8..c2b93d2bafd 100644 --- a/ydb/core/base/path.h +++ b/ydb/core/base/path.h @@ -5,7 +5,7 @@ #include <util/string/join.h> namespace NKikimr { - + TVector<TString> SplitPath(TString path); TString JoinPath(const TVector<TString>& path); TString CanonizePath(const TString &path); @@ -19,15 +19,15 @@ bool IsEqualPaths(const TString& l, const TString& r) noexcept; bool IsStartWithSlash(const TString& l); bool CheckDbPath(const TString &path, const TString &domain, TString &error); TString::const_iterator PathPartBrokenAt(const TString &part, const TStringBuf extraSymbols = {}); -bool TrySplitPathByDb(const TString& path, const TString& database, - std::pair<TString, TString>& result, TString& error); - -template <typename TIter> -TString CombinePath(TIter begin, TIter end, bool canonize = true) { - auto path = JoinRange("/", begin, end); - return canonize - ? CanonizePath(path) - : path; +bool TrySplitPathByDb(const TString& path, const TString& database, + std::pair<TString, TString>& result, TString& error); + +template <typename TIter> +TString CombinePath(TIter begin, TIter end, bool canonize = true) { + auto path = JoinRange("/", begin, end); + return canonize + ? CanonizePath(path) + : path; +} + } - -} diff --git a/ydb/core/base/pathid.cpp b/ydb/core/base/pathid.cpp index 553cc085333..0ad8a2d93cf 100644 --- a/ydb/core/base/pathid.cpp +++ b/ydb/core/base/pathid.cpp @@ -1,52 +1,52 @@ -#include "pathid.h" - -#include <util/stream/str.h> - -namespace NKikimr { - -ui64 TPathId::Hash() const { - return Hash128to32(OwnerId, LocalPathId); -} - -TString TPathId::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TPathId::Out(IOutputStream& o) const { - if (!*this) { - o << "<Invalid>"; - } else { - o << "[OwnerId: " << OwnerId << ", LocalPathId: " << LocalPathId << "]"; - } -} - -bool TPathId::operator<(const TPathId& x) const { - return OwnerId != x.OwnerId ? OwnerId < x.OwnerId : LocalPathId < x.LocalPathId; -} - -bool TPathId::operator>(const TPathId& x) const { - return x < *this; -} - -bool TPathId::operator<=(const TPathId& x) const { - return OwnerId != x.OwnerId ? OwnerId < x.OwnerId : LocalPathId <= x.LocalPathId; -} - -bool TPathId::operator>=(const TPathId& x) const { - return x <= *this; -} - -bool TPathId::operator==(const TPathId& x) const { - return OwnerId == x.OwnerId && LocalPathId == x.LocalPathId; -} - -bool TPathId::operator!=(const TPathId& x) const { - return OwnerId != x.OwnerId || LocalPathId != x.LocalPathId; -} - +#include "pathid.h" + +#include <util/stream/str.h> + +namespace NKikimr { + +ui64 TPathId::Hash() const { + return Hash128to32(OwnerId, LocalPathId); +} + +TString TPathId::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TPathId::Out(IOutputStream& o) const { + if (!*this) { + o << "<Invalid>"; + } else { + o << "[OwnerId: " << OwnerId << ", LocalPathId: " << LocalPathId << "]"; + } +} + +bool TPathId::operator<(const TPathId& x) const { + return OwnerId != x.OwnerId ? OwnerId < x.OwnerId : LocalPathId < x.LocalPathId; +} + +bool TPathId::operator>(const TPathId& x) const { + return x < *this; +} + +bool TPathId::operator<=(const TPathId& x) const { + return OwnerId != x.OwnerId ? OwnerId < x.OwnerId : LocalPathId <= x.LocalPathId; +} + +bool TPathId::operator>=(const TPathId& x) const { + return x <= *this; +} + +bool TPathId::operator==(const TPathId& x) const { + return OwnerId == x.OwnerId && LocalPathId == x.LocalPathId; +} + +bool TPathId::operator!=(const TPathId& x) const { + return OwnerId != x.OwnerId || LocalPathId != x.LocalPathId; +} + TPathId TPathId::NextId() const { if (!this->operator bool()) { return TPathId(); @@ -79,17 +79,17 @@ TPathId TPathId::PrevId() const { return TPathId(OwnerId, LocalPathId - 1); } -TPathId::operator bool() const { - return OwnerId != InvalidOwnerId && LocalPathId != InvalidLocalPathId; -} - -TPathId PathIdFromPathId(const NKikimrProto::TPathID& proto) { - return TPathId(proto.GetOwnerId(), proto.GetLocalId()); -} - -void PathIdFromPathId(const TPathId& pathId, NKikimrProto::TPathID* proto) { - proto->SetOwnerId(pathId.OwnerId); - proto->SetLocalId(pathId.LocalPathId); -} - -} // NKikimr +TPathId::operator bool() const { + return OwnerId != InvalidOwnerId && LocalPathId != InvalidLocalPathId; +} + +TPathId PathIdFromPathId(const NKikimrProto::TPathID& proto) { + return TPathId(proto.GetOwnerId(), proto.GetLocalId()); +} + +void PathIdFromPathId(const TPathId& pathId, NKikimrProto::TPathID* proto) { + proto->SetOwnerId(pathId.OwnerId); + proto->SetLocalId(pathId.LocalPathId); +} + +} // NKikimr diff --git a/ydb/core/base/pathid.h b/ydb/core/base/pathid.h index 9f989fc64ce..e974f846134 100644 --- a/ydb/core/base/pathid.h +++ b/ydb/core/base/pathid.h @@ -1,21 +1,21 @@ -#pragma once - -#include "defs.h" - -#include <ydb/core/protos/base.pb.h> - -namespace NKikimr { - -using TOwnerId = ui64; +#pragma once + +#include "defs.h" + +#include <ydb/core/protos/base.pb.h> + +namespace NKikimr { + +using TOwnerId = ui64; constexpr TOwnerId InvalidOwnerId = Max<ui64>(); - -using TLocalPathId = ui64; + +using TLocalPathId = ui64; constexpr TLocalPathId InvalidLocalPathId = Max<ui64>(); - -struct TPathId { - TOwnerId OwnerId; - TLocalPathId LocalPathId; - + +struct TPathId { + TOwnerId OwnerId; + TLocalPathId LocalPathId; + constexpr TPathId() : OwnerId(InvalidOwnerId) , LocalPathId(InvalidLocalPathId) @@ -25,36 +25,36 @@ struct TPathId { , LocalPathId(localPathId) { } - - ui64 Hash() const; - TString ToString() const; - void Out(IOutputStream& o) const; - - bool operator<(const TPathId& x) const; - bool operator>(const TPathId& x) const; - bool operator<=(const TPathId& x) const; - bool operator>=(const TPathId& x) const; - bool operator==(const TPathId& x) const; - bool operator!=(const TPathId& x) const; + + ui64 Hash() const; + TString ToString() const; + void Out(IOutputStream& o) const; + + bool operator<(const TPathId& x) const; + bool operator>(const TPathId& x) const; + bool operator<=(const TPathId& x) const; + bool operator>=(const TPathId& x) const; + bool operator==(const TPathId& x) const; + bool operator!=(const TPathId& x) const; explicit operator bool() const; - + TPathId NextId() const; TPathId PrevId() const; -}; // TPathId - -TPathId PathIdFromPathId(const NKikimrProto::TPathID& proto); -void PathIdFromPathId(const TPathId& pathId, NKikimrProto::TPathID* proto); - -} // NKikimr - -template<> -struct THash<NKikimr::TPathId> { - inline ui64 operator()(const NKikimr::TPathId& x) const noexcept { - return x.Hash(); - } -}; - +}; // TPathId + +TPathId PathIdFromPathId(const NKikimrProto::TPathID& proto); +void PathIdFromPathId(const TPathId& pathId, NKikimrProto::TPathID* proto); + +} // NKikimr + +template<> +struct THash<NKikimr::TPathId> { + inline ui64 operator()(const NKikimr::TPathId& x) const noexcept { + return x.Hash(); + } +}; + namespace std { template <> struct hash<NKikimr::TPathId> { @@ -64,7 +64,7 @@ struct hash<NKikimr::TPathId> { }; } -template<> -inline void Out<NKikimr::TPathId>(IOutputStream& o, const NKikimr::TPathId& x) { - return x.Out(o); -} +template<> +inline void Out<NKikimr::TPathId>(IOutputStream& o, const NKikimr::TPathId& x) { + return x.Out(o); +} diff --git a/ydb/core/base/statestorage.h b/ydb/core/base/statestorage.h index af68404ddc1..9b3e3b6b1b7 100644 --- a/ydb/core/base/statestorage.h +++ b/ydb/core/base/statestorage.h @@ -22,8 +22,8 @@ struct TEvStateStorage { EvCleanup, EvResolveBoard, EvBoardInfo, - EvResolveSchemeBoard, // subset (by hash) - EvListSchemeBoard, // all + EvResolveSchemeBoard, // subset (by hash) + EvListSchemeBoard, // all EvUpdateGroupConfig, // replies (local, from proxy) @@ -32,7 +32,7 @@ struct TEvStateStorage { EvResolveReplicasList, EvResponseReplicasDumps, EvDeleteResult, - EvListSchemeBoardResult, + EvListSchemeBoardResult, // replicas interface EvReplicaLookup = EvLock + 2 * 512, @@ -358,7 +358,7 @@ struct TEvStateStorage { struct TEvResolveReplicas; struct TEvResolveBoard; - struct TEvResolveSchemeBoard; + struct TEvResolveSchemeBoard; struct TEvResolveReplicasList; struct TEvReplicaLookup; struct TEvReplicaInfo; @@ -371,8 +371,8 @@ struct TEvStateStorage { struct TEvReplicaBoardCleanup; struct TEvReplicaBoardPublishAck; struct TEvReplicaBoardInfo; - struct TEvListSchemeBoard; - struct TEvListSchemeBoardResult; + struct TEvListSchemeBoard; + struct TEvListSchemeBoardResult; struct TEvUpdateGroupConfig; struct TEvReplicaProbeSubscribe; struct TEvReplicaProbeUnsubscribe; @@ -514,7 +514,7 @@ void BuildStateStorageInfos(const NKikimrConfig::TDomainsConfig::TStateStorage& IActor* CreateStateStorageWarden(const TIntrusivePtr<TStateStorageInfo> &info, const TIntrusivePtr<TStateStorageInfo> &board, const TIntrusivePtr<TStateStorageInfo> &schemeBoard); -IActor* CreateStateStorageProxy(const TIntrusivePtr<TStateStorageInfo> &info, const TIntrusivePtr<TStateStorageInfo> &board, const TIntrusivePtr<TStateStorageInfo> &schemeBoard); +IActor* CreateStateStorageProxy(const TIntrusivePtr<TStateStorageInfo> &info, const TIntrusivePtr<TStateStorageInfo> &board, const TIntrusivePtr<TStateStorageInfo> &schemeBoard); IActor* CreateStateStorageProxyStub(); IActor* CreateStateStorageReplica(const TIntrusivePtr<TStateStorageInfo> &info, ui32 replicaIndex); IActor* CreateStateStorageMonitoringActor(ui64 targetTablet, const TActorId &sender, const TString &query); diff --git a/ydb/core/base/statestorage_guardian.cpp b/ydb/core/base/statestorage_guardian.cpp index 3efacbdad73..f1df97896b5 100644 --- a/ydb/core/base/statestorage_guardian.cpp +++ b/ydb/core/base/statestorage_guardian.cpp @@ -2,16 +2,16 @@ #include "statestorage_guardian_impl.h" #include "tabletid.h" #include "tablet.h" - + #include <ydb/core/base/appdata.h> #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> - -#include <util/generic/algorithm.h> + +#include <util/generic/algorithm.h> #include <util/generic/xrange.h> namespace NKikimr { diff --git a/ydb/core/base/statestorage_impl.h b/ydb/core/base/statestorage_impl.h index 5e1b65699e8..2a2aa22f6a7 100644 --- a/ydb/core/base/statestorage_impl.h +++ b/ydb/core/base/statestorage_impl.h @@ -1,5 +1,5 @@ #pragma once -#include "pathid.h" +#include "pathid.h" #include "statestorage.h" namespace NKikimr { @@ -127,44 +127,44 @@ struct TEvStateStorage::TEvResolveBoard : public TEventLocal<TEvResolveBoard, Ev {} }; -struct TEvStateStorage::TEvResolveSchemeBoard : public TEventLocal<TEvResolveSchemeBoard, EvResolveSchemeBoard> { - enum EKeyType { - KeyTypePath, - KeyTypePathId, - }; - - const TString Path; - const TPathId PathId; - - const EKeyType KeyType; - - TEvResolveSchemeBoard(const TString &path) - : Path(path) - , KeyType(KeyTypePath) - {} - - TEvResolveSchemeBoard(const TPathId& pathId) - : PathId(pathId) - , KeyType(KeyTypePathId) - {} -}; - +struct TEvStateStorage::TEvResolveSchemeBoard : public TEventLocal<TEvResolveSchemeBoard, EvResolveSchemeBoard> { + enum EKeyType { + KeyTypePath, + KeyTypePathId, + }; + + const TString Path; + const TPathId PathId; + + const EKeyType KeyType; + + TEvResolveSchemeBoard(const TString &path) + : Path(path) + , KeyType(KeyTypePath) + {} + + TEvResolveSchemeBoard(const TPathId& pathId) + : PathId(pathId) + , KeyType(KeyTypePathId) + {} +}; + struct TEvStateStorage::TEvResolveReplicasList : public TEventLocal<TEvResolveReplicasList, EvResolveReplicasList> { TVector<TActorId> Replicas; ui32 ConfigContentHash = Max<ui32>(); }; -struct TEvStateStorage::TEvListSchemeBoard : public TEventLocal<TEvListSchemeBoard, EvListSchemeBoard> { -}; - -struct TEvStateStorage::TEvListSchemeBoardResult : public TEventLocal<TEvListSchemeBoardResult, EvListSchemeBoardResult> { +struct TEvStateStorage::TEvListSchemeBoard : public TEventLocal<TEvListSchemeBoard, EvListSchemeBoard> { +}; + +struct TEvStateStorage::TEvListSchemeBoardResult : public TEventLocal<TEvListSchemeBoardResult, EvListSchemeBoardResult> { TIntrusiveConstPtr<TStateStorageInfo> Info; TEvListSchemeBoardResult(const TIntrusiveConstPtr<TStateStorageInfo> &info) : Info(info) {} -}; - +}; + struct TEvStateStorage::TEvReplicaLookup : public TEventPB<TEvStateStorage::TEvReplicaLookup, NKikimrStateStorage::TEvLookup, TEvStateStorage::EvReplicaLookup>{ struct TActualityCounter : public TRefCounted<TActualityCounter, TAtomicCounter> {}; using TActualityCounterPtr = TIntrusivePtr<TActualityCounter>; diff --git a/ydb/core/base/statestorage_proxy.cpp b/ydb/core/base/statestorage_proxy.cpp index 792a5ed44c0..13d396cb0f1 100644 --- a/ydb/core/base/statestorage_proxy.cpp +++ b/ydb/core/base/statestorage_proxy.cpp @@ -1,15 +1,15 @@ #include "statestorage_impl.h" #include "tabletid.h" - + #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/helpers/flow_controlled_queue.h> - + #include <util/digest/city.h> #include <util/generic/xrange.h> @@ -736,7 +736,7 @@ public: class TStateStorageProxy : public TActor<TStateStorageProxy> { TIntrusivePtr<TStateStorageInfo> Info; TIntrusivePtr<TStateStorageInfo> BoardInfo; - TIntrusivePtr<TStateStorageInfo> SchemeBoardInfo; + TIntrusivePtr<TStateStorageInfo> SchemeBoardInfo; TIntrusivePtr<TStateStorageInfo> FlowControlledInfo; @@ -763,55 +763,55 @@ class TStateStorageProxy : public TActor<TStateStorageProxy> { } void Handle(TEvStateStorage::TEvResolveReplicas::TPtr &ev) { - ResolveReplicas(ev, ev->Get()->TabletID, Info); - } - - void Handle(TEvStateStorage::TEvResolveBoard::TPtr &ev) { - if (!BoardInfo) { - Send(ev->Sender, new TEvStateStorage::TEvResolveReplicasList(), 0, ev->Cookie); - return; - } - - const auto *msg = ev->Get(); - const ui64 pathHash = CityHash64(msg->Path); - - ResolveReplicas(ev, pathHash, BoardInfo); - } - - void Handle(TEvStateStorage::TEvResolveSchemeBoard::TPtr &ev) { - if (!SchemeBoardInfo) { - Send(ev->Sender, new TEvStateStorage::TEvResolveReplicasList(), 0, ev->Cookie); - return; - } - - const auto *msg = ev->Get(); - + ResolveReplicas(ev, ev->Get()->TabletID, Info); + } + + void Handle(TEvStateStorage::TEvResolveBoard::TPtr &ev) { + if (!BoardInfo) { + Send(ev->Sender, new TEvStateStorage::TEvResolveReplicasList(), 0, ev->Cookie); + return; + } + + const auto *msg = ev->Get(); + const ui64 pathHash = CityHash64(msg->Path); + + ResolveReplicas(ev, pathHash, BoardInfo); + } + + void Handle(TEvStateStorage::TEvResolveSchemeBoard::TPtr &ev) { + if (!SchemeBoardInfo) { + Send(ev->Sender, new TEvStateStorage::TEvResolveReplicasList(), 0, ev->Cookie); + return; + } + + const auto *msg = ev->Get(); + ui64 fakeTabletId; - switch (msg->KeyType) { - case TEvStateStorage::TEvResolveSchemeBoard::KeyTypePath: + switch (msg->KeyType) { + case TEvStateStorage::TEvResolveSchemeBoard::KeyTypePath: fakeTabletId = CityHash64(msg->Path); - break; - - case TEvStateStorage::TEvResolveSchemeBoard::KeyTypePathId: + break; + + case TEvStateStorage::TEvResolveSchemeBoard::KeyTypePathId: fakeTabletId = msg->PathId.Hash(); - break; - - default: - Y_FAIL("unreachable"); - } - + break; + + default: + Y_FAIL("unreachable"); + } + ResolveReplicas(ev, fakeTabletId, SchemeBoardInfo); - } - - void Handle(TEvStateStorage::TEvListSchemeBoard::TPtr &ev) { - if (!SchemeBoardInfo) { + } + + void Handle(TEvStateStorage::TEvListSchemeBoard::TPtr &ev) { + if (!SchemeBoardInfo) { Send(ev->Sender, new TEvStateStorage::TEvListSchemeBoardResult(nullptr), 0, ev->Cookie); - return; - } - + return; + } + Send(ev->Sender, new TEvStateStorage::TEvListSchemeBoardResult(SchemeBoardInfo), 0, ev->Cookie); - } - + } + void Handle(TEvStateStorage::TEvReplicaProbeSubscribe::TPtr &ev) { const auto *msg = ev->Get(); @@ -872,8 +872,8 @@ class TStateStorageProxy : public TActor<TStateStorageProxy> { } } - template<typename TEventPtr> - void ResolveReplicas(const TEventPtr &ev, ui64 tabletId, const TIntrusivePtr<TStateStorageInfo> &info) const { + template<typename TEventPtr> + void ResolveReplicas(const TEventPtr &ev, ui64 tabletId, const TIntrusivePtr<TStateStorageInfo> &info) const { THolder<TStateStorageInfo::TSelection> selection(new TStateStorageInfo::TSelection()); info->SelectReplicas(tabletId, selection.Get()); @@ -950,14 +950,14 @@ public: return NKikimrServices::TActivity::SS_PROXY; } - TStateStorageProxy( - const TIntrusivePtr<TStateStorageInfo> &info, - const TIntrusivePtr<TStateStorageInfo> &boardInfo, - const TIntrusivePtr<TStateStorageInfo> &schemeBoardInfo) + TStateStorageProxy( + const TIntrusivePtr<TStateStorageInfo> &info, + const TIntrusivePtr<TStateStorageInfo> &boardInfo, + const TIntrusivePtr<TStateStorageInfo> &schemeBoardInfo) : TActor(&TThis::StateInit) , Info(info) , BoardInfo(boardInfo) - , SchemeBoardInfo(schemeBoardInfo) + , SchemeBoardInfo(schemeBoardInfo) {} STATEFN(StateInit) { @@ -970,8 +970,8 @@ public: hFunc(TEvStateStorage::TEvCleanup, Handle); hFunc(TEvStateStorage::TEvResolveReplicas, Handle); hFunc(TEvStateStorage::TEvResolveBoard, Handle); - hFunc(TEvStateStorage::TEvResolveSchemeBoard, Handle); - hFunc(TEvStateStorage::TEvListSchemeBoard, Handle); + hFunc(TEvStateStorage::TEvResolveSchemeBoard, Handle); + hFunc(TEvStateStorage::TEvListSchemeBoard, Handle); hFunc(TEvStateStorage::TEvUpdateGroupConfig, Handle); hFunc(TEvStateStorage::TEvReplicaProbeSubscribe, Handle); hFunc(TEvStateStorage::TEvReplicaProbeUnsubscribe, Handle); @@ -1017,12 +1017,12 @@ public: } }; -IActor* CreateStateStorageProxy( - const TIntrusivePtr<TStateStorageInfo> &info, - const TIntrusivePtr<TStateStorageInfo> &board, - const TIntrusivePtr<TStateStorageInfo> &schemeBoard -) { - return new TStateStorageProxy(info, board, schemeBoard); +IActor* CreateStateStorageProxy( + const TIntrusivePtr<TStateStorageInfo> &info, + const TIntrusivePtr<TStateStorageInfo> &board, + const TIntrusivePtr<TStateStorageInfo> &schemeBoard +) { + return new TStateStorageProxy(info, board, schemeBoard); } IActor* CreateStateStorageProxyStub() { diff --git a/ydb/core/base/tablet_types.h b/ydb/core/base/tablet_types.h index 72e958762bc..f585819299c 100644 --- a/ydb/core/base/tablet_types.h +++ b/ydb/core/base/tablet_types.h @@ -40,7 +40,7 @@ public: static constexpr EType FILESTORE = FileStore; static constexpr EType TESTSHARD = TestShard; static constexpr EType SEQUENCESHARD = SequenceShard; - static constexpr EType REPLICATION_CONTROLLER = ReplicationController; + static constexpr EType REPLICATION_CONTROLLER = ReplicationController; static const char* TypeToStr(EType t) { return EType_Name(t).c_str(); @@ -58,6 +58,6 @@ public: } // end of NKikimr -Y_DECLARE_OUT_SPEC(inline, NKikimrTabletBase::TTabletTypes::EType, os, type) { +Y_DECLARE_OUT_SPEC(inline, NKikimrTabletBase::TTabletTypes::EType, os, type) { os << NKikimrTabletBase::TTabletTypes::EType_Name(type); } diff --git a/ydb/core/base/ya.make b/ydb/core/base/ya.make index bff630a029c..83db5825c3a 100644 --- a/ydb/core/base/ya.make +++ b/ydb/core/base/ya.make @@ -43,7 +43,7 @@ SRCS( logoblob.h nameservice.h path.cpp - pathid.cpp + pathid.cpp pool_stats_collector.cpp pool_stats_collector.h quoter.cpp diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h index 0d6b560dc7f..a3d764aa7e1 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h @@ -9,7 +9,7 @@ #include <ydb/core/control/immediate_control_board_wrapper.h> #include <ydb/core/protos/blobstorage.pb.h> #include <ydb/core/protos/config.pb.h> - + #include <ydb/library/pdisk_io/file_params.h> #include <ydb/library/pdisk_io/sector_map.h> diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h index a1de1954114..6b818645dd9 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h @@ -1,13 +1,13 @@ #pragma once #include "defs.h" - + #include "blobstorage_pdisk.h" #include "blobstorage_pdisk_crypto.h" #include "blobstorage_pdisk_data.h" #include "blobstorage_pdisk_drivemodel.h" #include "blobstorage_pdisk_mon.h" #include "blobstorage_pdisk_request_id.h" - + #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/blobstorage/crypto/crypto.h> diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h index cf8bffa2931..47fbd959c96 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_events.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_events.h @@ -18,10 +18,10 @@ #include <ydb/core/util/pb.h> #include <library/cpp/string_utils/base64/base64.h> - + #include <util/digest/multi.h> #include <util/generic/maybe.h> -#include <util/stream/str.h> +#include <util/stream/str.h> #include <util/string/escape.h> // FIXME: this check is obsolete; TEvVGet message generators check expected reply size by their own diff --git a/ydb/core/client/client_ut.cpp b/ydb/core/client/client_ut.cpp index 2577f233fba..b94c722588b 100644 --- a/ydb/core/client/client_ut.cpp +++ b/ydb/core/client/client_ut.cpp @@ -15,10 +15,10 @@ #include <ydb/library/yql/ast/yql_ast.h> #include <ydb/library/yql/ast/yql_expr.h> - + #include <util/folder/path.h> -#include <util/generic/xrange.h> -#include <util/string/subst.h> +#include <util/generic/xrange.h> +#include <util/string/subst.h> #include <util/thread/pool.h> namespace NKikimr { diff --git a/ydb/core/client/flat_ut.cpp b/ydb/core/client/flat_ut.cpp index 78e93608298..362c052aa11 100644 --- a/ydb/core/client/flat_ut.cpp +++ b/ydb/core/client/flat_ut.cpp @@ -1931,7 +1931,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { annoyingClient.InitRoot(); annoyingClient.MkDir("/dc-1", "Dir"); - for (int colCount = 1; colCount < 100; ++colCount) { + for (int colCount = 1; colCount < 100; ++colCount) { NKikimrSchemeOp::TTableDescription schema; TString name = "Table_" + ToString(colCount); schema.SetName(name); @@ -1943,8 +1943,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) { } schema.AddKeyColumnNames("col_0"); - while (annoyingClient.CreateTable("/dc-1/Dir", schema) != NMsgBusProxy::MSTATUS_OK) {} - while (annoyingClient.CreateTable("/dc-1/Dir", " Name: '" + name + "_Copy' CopyFromTable: '/dc-1/Dir/" + name + "'") != NMsgBusProxy::MSTATUS_OK) {} + while (annoyingClient.CreateTable("/dc-1/Dir", schema) != NMsgBusProxy::MSTATUS_OK) {} + while (annoyingClient.CreateTable("/dc-1/Dir", " Name: '" + name + "_Copy' CopyFromTable: '/dc-1/Dir/" + name + "'") != NMsgBusProxy::MSTATUS_OK) {} auto fnGetColumns = [](const NMsgBusProxy::TBusResponse* lsRes) { UNIT_ASSERT(lsRes); @@ -3161,7 +3161,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { Y_UNIT_TEST(AutoSplitMergeQueue) { TPortManager pm; ui16 port = pm.GetPort(2134); - TServer cleverServer = TServer(TServerSettings(port).SetEnableMvcc(false)); + TServer cleverServer = TServer(TServerSettings(port).SetEnableMvcc(false)); DisableSplitMergePartCountLimit(cleverServer); cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::OPS_COMPACT, NActors::NLog::PRI_ERROR); diff --git a/ydb/core/client/minikql_compile/mkql_compile_service.cpp b/ydb/core/client/minikql_compile/mkql_compile_service.cpp index cb019d2ccfb..e01d61a22cc 100644 --- a/ydb/core/client/minikql_compile/mkql_compile_service.cpp +++ b/ydb/core/client/minikql_compile/mkql_compile_service.cpp @@ -82,8 +82,8 @@ public: } NYql::IDbSchemeResolver* MakeDbSchemeResolver(const TActorContext& ctx) { - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), Counters); - SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), Counters); + SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); return NSchCache::CreateDbSchemeResolver(ctx.ExecutorThread.ActorSystem, SchemeCache); } diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index c37de19492c..76f38bf35cb 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -1437,9 +1437,9 @@ public: , DbSchemeResolver(dbSchemeResolver) , ResponseTo(responseTo) , ResolveRefreshCookies(std::move(resolveRefreshCookies)) - { - Y_UNUSED(forceCacheRefresh); - } + { + Y_UNUSED(forceCacheRefresh); + } void Bootstrap(const TActorContext& ctx) { auto* appData = AppData(ctx); diff --git a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp index 4e15e2f6d59..eb74fa36e94 100644 --- a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp +++ b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp @@ -44,7 +44,7 @@ class TTableProxyActor : public TActorBootstrapped<TTableProxyActor> { reply.Table.TableName = src.TableName; reply.Table.ColumnNames = std::move(src.ColumnNames); reply.TableId = new TTableId(res.TableId); - reply.CacheGeneration = 0; + reply.CacheGeneration = 0; TSet<TString> replyColumns; @@ -116,7 +116,7 @@ public: auto &x = request->ResultSet.back(); x.Path = SplitPath(table.TableName); x.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - x.ShowPrivatePath = true; + x.ShowPrivatePath = true; } ctx.Send(SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); diff --git a/ydb/core/client/server/msgbus_server_db.cpp b/ydb/core/client/server/msgbus_server_db.cpp index 7e7fdd60211..08f3d717864 100644 --- a/ydb/core/client/server/msgbus_server_db.cpp +++ b/ydb/core/client/server/msgbus_server_db.cpp @@ -236,7 +236,7 @@ protected: const NSchemeCache::TSchemeCacheNavigate& request = *ev->Get()->Request; Y_VERIFY(request.ResultSet.size() == 1); if (request.ResultSet.front().Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - return ReplyWithError(MSTATUS_ERROR, TEvTxUserProxy::TResultStatus::ResolveError, ToString(request.ResultSet.front().Status), ctx); + return ReplyWithError(MSTATUS_ERROR, TEvTxUserProxy::TResultStatus::ResolveError, ToString(request.ResultSet.front().Status), ctx); } CacheNavigate = ev->Get()->Request; if (++Responses == Requests) { diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp index 1fceca51ad2..f44ab6f480f 100644 --- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp @@ -18,8 +18,8 @@ namespace NPqMetaCacheV2 { IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { auto appData = ActorSystem->AppData<TAppData>(); - auto cacheCounters = GetServiceCounters(counters, "pqproxy|schemecache"); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); + auto cacheCounters = GetServiceCounters(counters, "pqproxy|schemecache"); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); return CreateSchemeBoardSchemeCache(cacheConfig.Get()); } @@ -49,7 +49,7 @@ public: void Bootstrap(const TActorContext& ctx) { if (ClientWrapper == nullptr) { - auto* driver = AppData(ctx)->YdbDriver; + auto* driver = AppData(ctx)->YdbDriver; if (driver == nullptr) { LOG_WARN_S( ctx, NKikimrServices::PQ_METACACHE, @@ -329,19 +329,19 @@ private: auto& waiter = waiterIter->second; if (waiter.Type == EWaiterType::DescribeAllTopics) { - for (const auto& entry : result->ResultSet) { - if (!entry.PQGroupInfo) { - continue; - } - - const auto& desc = entry.PQGroupInfo->Description; - if (desc.HasBalancerTabletID() && desc.GetBalancerTabletID() != 0) { - continue; - } - - FullTopicsCacheOutdated = true; - } - + for (const auto& entry : result->ResultSet) { + if (!entry.PQGroupInfo) { + continue; + } + + const auto& desc = entry.PQGroupInfo->Description; + if (desc.HasBalancerTabletID() && desc.GetBalancerTabletID() != 0) { + continue; + } + + FullTopicsCacheOutdated = true; + } + FullTopicsCache.Reset(result); LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Updated topics cache with " << FullTopicsCache->ResultSet.size()); SendDescribeAllTopicsResponse(waiter.WaiterId, ctx); diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index 7d55b3c3a82..237bba147b9 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -161,7 +161,7 @@ protected: THolder<TEvPersQueue::TEvUpdateConfig> request(new TEvPersQueue::TEvUpdateConfig()); for (size_t i : partitions) { - request->Record.MutableTabletConfig()->AddPartitionIds(i); + request->Record.MutableTabletConfig()->AddPartitionIds(i); } request->Record.MutableTabletConfig()->SetCacheSize(10*1024*1024); request->Record.SetTxId(12345); diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 4aa7f5f689e..7f3146d21a7 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -174,8 +174,8 @@ void TMessageBusServerProxy::Bootstrap(const TActorContext& ctx) { SchemeCacheCounters = GetServiceCounters(AppData(ctx)->Counters, "pqproxy|cache"); DbOperationsCounters = new TMessageBusDbOpsCounters(AppData(ctx)->Counters); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), SchemeCacheCounters); - SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), SchemeCacheCounters); + SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); PqMetaCache = CreatePersQueueMetaCacheV2Id(); if (Server) { diff --git a/ydb/core/client/server/msgbus_server_s3_listing.cpp b/ydb/core/client/server/msgbus_server_s3_listing.cpp index 3f44e57b3a6..3626866322b 100644 --- a/ydb/core/client/server/msgbus_server_s3_listing.cpp +++ b/ydb/core/client/server/msgbus_server_s3_listing.cpp @@ -158,7 +158,7 @@ private: Y_VERIFY(request.ResultSet.size() == 1); if (request.ResultSet.front().Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { return ReplyWithError(MSTATUS_ERROR, NTxProxy::TResultStatus::EStatus::ResolveError, - ToString(request.ResultSet.front().Status), ctx); + ToString(request.ResultSet.front().Status), ctx); } ResolveNamesResult = ev->Get()->Request; diff --git a/ydb/core/client/server/pq_metacache_v2_ut.cpp b/ydb/core/client/server/pq_metacache_v2_ut.cpp index cd96f4883ab..f02fd420c9a 100644 --- a/ydb/core/client/server/pq_metacache_v2_ut.cpp +++ b/ydb/core/client/server/pq_metacache_v2_ut.cpp @@ -71,10 +71,10 @@ class TPqMetaCacheV2Test: public TTestBase { EdgeActorId = runtime->AllocateEdgeActor(); auto config = MakeIntrusive<TSchemeCacheConfig>(); config->Counters = new NMonitoring::TDynamicCounters; - config->Roots.emplace_back(settings.Domain, 8716544, "Root"); + config->Roots.emplace_back(settings.Domain, 8716544, "Root"); auto& domainsInfo = settings.AppConfig.GetDomainsConfig(); for (const auto& domain : domainsInfo.GetDomain()) { - config->Roots.emplace_back(domain.GetDomainId(), domain.GetSchemeRoot(), domain.GetName()); + config->Roots.emplace_back(domain.GetDomainId(), domain.GetSchemeRoot(), domain.GetName()); } SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get())); MetaCacheId = runtime->Register( diff --git a/ydb/core/cms/audit_log.cpp b/ydb/core/cms/audit_log.cpp index b9eee5bd1c5..d1827c117d1 100644 --- a/ydb/core/cms/audit_log.cpp +++ b/ydb/core/cms/audit_log.cpp @@ -1,40 +1,40 @@ -#include "audit_log.h" - -#include <library/cpp/actors/core/log.h> - -#include <util/string/builder.h> - -namespace NKikimr { -namespace NCms { - -void AuditLog(const TString& component, const TString& message, const TActorContext& ctx) { - LOG_NOTICE_S(ctx, NKikimrServices::CMS, "[AuditLog] [" << component << "] " << message); -} - -void AuditLog(const TString& component, const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { - const TString message = TStringBuilder() << "Reply" - << ": request# " << request->ToString() - << ", response# " << response->ToString(); - - AuditLog(component, message, ctx); -} - -void AuditLog(const TString& component, NMon::TEvHttpInfo::TPtr& request, const TString& response, const TActorContext& ctx) { - const auto& r = request->Get()->Request; - - const TString requestStr = TStringBuilder() << "{" - << " method: " << r.GetMethod() - << " path: " << r.GetPath() - << " addr: " << r.GetRemoteAddr() - << " body: " << r.GetPostContent() - << " }"; - - const TString message = TStringBuilder() << "Reply" - << ": request# " << requestStr.Quote() - << ", response# " << response.Quote(); - - AuditLog(component, message, ctx); -} - -} // NCms -} // NKikimr +#include "audit_log.h" + +#include <library/cpp/actors/core/log.h> + +#include <util/string/builder.h> + +namespace NKikimr { +namespace NCms { + +void AuditLog(const TString& component, const TString& message, const TActorContext& ctx) { + LOG_NOTICE_S(ctx, NKikimrServices::CMS, "[AuditLog] [" << component << "] " << message); +} + +void AuditLog(const TString& component, const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { + const TString message = TStringBuilder() << "Reply" + << ": request# " << request->ToString() + << ", response# " << response->ToString(); + + AuditLog(component, message, ctx); +} + +void AuditLog(const TString& component, NMon::TEvHttpInfo::TPtr& request, const TString& response, const TActorContext& ctx) { + const auto& r = request->Get()->Request; + + const TString requestStr = TStringBuilder() << "{" + << " method: " << r.GetMethod() + << " path: " << r.GetPath() + << " addr: " << r.GetRemoteAddr() + << " body: " << r.GetPostContent() + << " }"; + + const TString message = TStringBuilder() << "Reply" + << ": request# " << requestStr.Quote() + << ", response# " << response.Quote(); + + AuditLog(component, message, ctx); +} + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/audit_log.h b/ydb/core/cms/audit_log.h index 9ebf6205d38..472e12f8be6 100644 --- a/ydb/core/cms/audit_log.h +++ b/ydb/core/cms/audit_log.h @@ -1,17 +1,17 @@ -#pragma once - -#include "defs.h" - -#include <library/cpp/actors/core/mon.h> - -namespace NKikimr { -namespace NCms { - -void AuditLog(const TString& component, const TString& message, const TActorContext& ctx); -void AuditLog(const TString& component, const IEventBase* request, const IEventBase* response, const TActorContext& ctx); - -// for http request/response -void AuditLog(const TString& component, NMon::TEvHttpInfo::TPtr& request, const TString& response, const TActorContext& ctx); - -} // NCms -} // NKikimr +#pragma once + +#include "defs.h" + +#include <library/cpp/actors/core/mon.h> + +namespace NKikimr { +namespace NCms { + +void AuditLog(const TString& component, const TString& message, const TActorContext& ctx); +void AuditLog(const TString& component, const IEventBase* request, const IEventBase* response, const TActorContext& ctx); + +// for http request/response +void AuditLog(const TString& component, NMon::TEvHttpInfo::TPtr& request, const TString& response, const TActorContext& ctx); + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/cluster_info.cpp b/ydb/core/cms/cluster_info.cpp index cc4135faa51..a1dc8a0b95f 100644 --- a/ydb/core/cms/cluster_info.cpp +++ b/ydb/core/cms/cluster_info.cpp @@ -1,7 +1,7 @@ #include "cluster_info.h" #include "cms_state.h" -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/system/hostname.h> #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR @@ -24,7 +24,7 @@ bool TLockableItem::IsLocked(TErrorInfo &error, TDuration defaultRetryTime, Y_VERIFY(Lock.Defined()); error.Code = TStatus::DISALLOW_TEMP; error.Reason = Sprintf("%s is restarting (permission %s owned by %s)", - PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); + PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); error.Deadline = Lock->ActionDeadline; return true; } @@ -32,7 +32,7 @@ bool TLockableItem::IsLocked(TErrorInfo &error, TDuration defaultRetryTime, if (Lock.Defined()) { error.Code = TStatus::DISALLOW_TEMP; error.Reason = Sprintf("%s has planned shutdown (permission %s owned by %s)", - PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); + PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); error.Deadline = Lock->ActionDeadline; return true; } @@ -47,7 +47,7 @@ bool TLockableItem::IsLocked(TErrorInfo &error, TDuration defaultRetryTime, error.Code = TStatus::DISALLOW_TEMP; error.Reason = Sprintf("%s has planned shutdown (notification %s owned by %s)", - PrettyItemName().data(), lock.NotificationId.data(), lock.Owner.data()); + PrettyItemName().data(), lock.NotificationId.data(), lock.Owner.data()); error.Deadline = lock.LockDeadline; return true; } @@ -55,7 +55,7 @@ bool TLockableItem::IsLocked(TErrorInfo &error, TDuration defaultRetryTime, if (!ScheduledLocks.empty() && ScheduledLocks.begin()->Order < DeactivatedLocksOrder) { error.Code = TStatus::DISALLOW_TEMP; error.Reason = Sprintf("%s has scheduled action %s owned by %s (order %" PRIu64 " vs %" PRIu64 ")", - PrettyItemName().data(), ScheduledLocks.begin()->RequestId.data(), + PrettyItemName().data(), ScheduledLocks.begin()->RequestId.data(), ScheduledLocks.begin()->Owner.data(), ScheduledLocks.begin()->Order, DeactivatedLocksOrder); error.Deadline = now + defaultRetryTime; @@ -64,7 +64,7 @@ bool TLockableItem::IsLocked(TErrorInfo &error, TDuration defaultRetryTime, if (!TempLocks.empty()) { error.Code = TStatus::DISALLOW; - error.Reason = Sprintf("%s has temporary lock", PrettyItemName().data()); + error.Reason = Sprintf("%s has temporary lock", PrettyItemName().data()); error.Deadline = now + defaultRetryTime; error.RollbackPoint = TempLocks.back().RollbackPoint; return true; @@ -79,14 +79,14 @@ bool TLockableItem::IsDown(TErrorInfo &error, TInstant defaultDeadline) const Y_VERIFY(Lock.Defined()); error.Code = TStatus::DISALLOW_TEMP; error.Reason = Sprintf("%s is restarting (permission %s owned by %s)", - PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); + PrettyItemName().data(), Lock->PermissionId.data(), Lock->Owner.data()); error.Deadline = Lock->ActionDeadline; return true; } if (State != UP) { error.Code = TStatus::DISALLOW_TEMP; - error.Reason = Sprintf("%s is down", PrettyItemName().data()); + error.Reason = Sprintf("%s is down", PrettyItemName().data()); error.Deadline = defaultDeadline; return true; } @@ -120,7 +120,7 @@ void TLockableItem::RemoveScheduledLocks(const TString &requestId) }); } -void TLockableItem::MigrateOldInfo(const TLockableItem &old) +void TLockableItem::MigrateOldInfo(const TLockableItem &old) { Downtime = old.Downtime; if (State != NKikimrCms::UP) @@ -143,9 +143,9 @@ void TLockableItem::DebugLocksDump(IOutputStream &ss, const TString &prefix) con ss << prefix << "Temporary lock at point " << lock.RollbackPoint; } -void TNodeInfo::MigrateOldInfo(const TLockableItem &old) +void TNodeInfo::MigrateOldInfo(const TLockableItem &old) { - TLockableItem::MigrateOldInfo(old); + TLockableItem::MigrateOldInfo(old); if (auto * oldNode = dynamic_cast<const TNodeInfo *>(&old)) { if (oldNode->State == UP) { PreviousTenant = oldNode->Tenant; @@ -164,27 +164,27 @@ TString TPDiskInfo::ItemName() const return Sprintf("PDisk %s", PDiskId.ToString().data()); } -TString TPDiskInfo::PrettyItemName() const -{ - TStringBuilder name; - - name << ItemName(); - if (Host || Path) { - name << " ("; - - if (Host) { - name << Host; - } - if (Path) { - name << ":" << Path; - } - - name << ")"; - } - - return name; -} - +TString TPDiskInfo::PrettyItemName() const +{ + TStringBuilder name; + + name << ItemName(); + if (Host || Path) { + name << " ("; + + if (Host) { + name << Host; + } + if (Path) { + name << ":" << Path; + } + + name << ")"; + } + + return name; +} + TString TPDiskInfo::GetDeviceName() const { return Sprintf("pdisk-%" PRIu32 "-%" PRIu32, PDiskId.NodeId, PDiskId.DiskId); @@ -216,9 +216,9 @@ TPDiskID TPDiskInfo::NameToId(const TString &name) return id; } -void TPDiskInfo::MigrateOldInfo(const TLockableItem &old) +void TPDiskInfo::MigrateOldInfo(const TLockableItem &old) { - TLockableItem::MigrateOldInfo(old); + TLockableItem::MigrateOldInfo(old); if (auto *oldPDisk = dynamic_cast<const TPDiskInfo *>(&old)) { if (!NodeId) NodeId = oldPDisk->NodeId; @@ -230,27 +230,27 @@ TString TVDiskInfo::ItemName() const return Sprintf("VDisk %s", VDiskId.ToString().data()); } -TString TVDiskInfo::PrettyItemName() const -{ - TStringBuilder name; - - name << ItemName(); - if (Host || Path) { - name << " ("; - - if (Host) { - name << Host; - } - if (Path) { - name << ":" << Path; - } - - name << ")"; - } - - return name; -} - +TString TVDiskInfo::PrettyItemName() const +{ + TStringBuilder name; + + name << ItemName(); + if (Host || Path) { + name << " ("; + + if (Host) { + name << Host; + } + if (Path) { + name << ":" << Path; + } + + name << ")"; + } + + return name; +} + TString TVDiskInfo::GetDeviceName() const { return Sprintf("vdisk-%u-%u-%u-%u-%u", VDiskId.GroupID, VDiskId.GroupGeneration, @@ -287,9 +287,9 @@ TVDiskID TVDiskInfo::NameToId(const TString &name) return id; } -void TVDiskInfo::MigrateOldInfo(const TLockableItem &old) +void TVDiskInfo::MigrateOldInfo(const TLockableItem &old) { - TLockableItem::MigrateOldInfo(old); + TLockableItem::MigrateOldInfo(old); if (auto *oldVDisk = dynamic_cast<const TVDiskInfo *>(&old)) { if (!NodeId) NodeId = oldVDisk->NodeId; @@ -305,7 +305,7 @@ void TClusterInfo::SetTimestamp(TInstant timestamp) entry.second->Timestamp = timestamp; } -void TClusterInfo::AddNode(const TEvInterconnect::TNodeInfo &info, const TActorContext *ctx) +void TClusterInfo::AddNode(const TEvInterconnect::TNodeInfo &info, const TActorContext *ctx) { TNodeInfoPtr &node = Nodes[info.NodeId]; if (!node) @@ -319,43 +319,43 @@ void TClusterInfo::AddNode(const TEvInterconnect::TNodeInfo &info, const TActorC node->Location = info.Location; node->State = NKikimrCms::UNKNOWN; - if (ctx) { - const auto maxStaticNodeId = AppData(*ctx)->DynamicNameserviceConfig->MaxStaticNodeId; - if (node->NodeId <= maxStaticNodeId) { - node->Services |= EService::Storage; - } else { - node->Services |= EService::DynamicNode; - } - } - + if (ctx) { + const auto maxStaticNodeId = AppData(*ctx)->DynamicNameserviceConfig->MaxStaticNodeId; + if (node->NodeId <= maxStaticNodeId) { + node->Services |= EService::Storage; + } else { + node->Services |= EService::DynamicNode; + } + } + auto range = HostNameToNodeId.equal_range(oldHost); - for (auto it = range.first; it != range.second; ++it) { + for (auto it = range.first; it != range.second; ++it) { if (it->second == node->NodeId) { HostNameToNodeId.erase(it); break; } - } - + } + HostNameToNodeId.emplace(node->Host, node->NodeId); LockableItems[node->ItemName()] = node; } -void TClusterInfo::SetNodeState(ui32 nodeId, NKikimrCms::EState state, const NKikimrWhiteboard::TSystemStateInfo &info) +void TClusterInfo::SetNodeState(ui32 nodeId, NKikimrCms::EState state, const NKikimrWhiteboard::TSystemStateInfo &info) { if (!HasNode(nodeId)) return; auto &node = NodeRef(nodeId); node.State = state; - node.Version = info.GetVersion(); - - node.Services = TServices(); - for (const auto& role : info.GetRoles()) { - EService value; - if (TryFromWhiteBoardRole(role, value)) { - node.Services |= value; - } - } + node.Version = info.GetVersion(); + + node.Services = TServices(); + for (const auto& role : info.GetRoles()) { + EService value; + if (TryFromWhiteBoardRole(role, value)) { + node.Services |= value; + } + } } void TClusterInfo::ClearNode(ui32 nodeId) @@ -427,7 +427,7 @@ void TClusterInfo::AddPDisk(const NKikimrBlobStorage::TBaseConfig::TPDisk &info) pdisk->Path = path; auto &node = NodeRef(nodeId); - pdisk->Host = node.Host; + pdisk->Host = node.Host; node.PDisks.insert(id); LockableItems[pdisk->ItemName()] = pdisk; @@ -474,12 +474,12 @@ void TClusterInfo::AddVDisk(const NKikimrBlobStorage::TBaseConfig::TVSlot &info) } auto &pdisk = PDiskRef(vdisk->PDiskId); - vdisk->Path = pdisk.Path; + vdisk->Path = pdisk.Path; pdisk.VDisks.insert(vdisk->VDiskId); pdisk.VSlots[vdisk->SlotId] = vdisk->VDiskId; auto &node = NodeRef(nodeId); - vdisk->Host = node.Host; + vdisk->Host = node.Host; node.VDisks.insert(vdisk->VDiskId); LockableItems[vdisk->ItemName()] = vdisk; @@ -488,10 +488,10 @@ void TClusterInfo::AddVDisk(const NKikimrBlobStorage::TBaseConfig::TVSlot &info) void TClusterInfo::UpdateVDiskState(const TVDiskID &id, const NKikimrWhiteboard::TVDiskStateInfo &info) { if (!HasVDisk(id)) { - if (IsStaticGroupVDisk(id)) { - return; - } - + if (IsStaticGroupVDisk(id)) { + return; + } + BLOG_ERROR("Cannot update state for unknown VDisk " << id.ToString()); return; } @@ -550,8 +550,8 @@ void TClusterInfo::AddNodeTenants(ui32 nodeId, const NKikimrTenantPool::TTenantP node.Tenant = nodeTenant; node.HasTenantInfo = true; - - TenantToNodeId.emplace(nodeTenant, nodeId); + + TenantToNodeId.emplace(nodeTenant, nodeId); } void TClusterInfo::AddNodeTempLock(ui32 nodeId, const NKikimrCms::TAction &action) @@ -572,23 +572,23 @@ void TClusterInfo::AddVDiskTempLock(TVDiskID vdiskId, const NKikimrCms::TAction vdisk.TempLocks.push_back({RollbackPoint, action}); } -static TServices MakeServices(const NKikimrCms::TAction &action) { - TServices services; - - if (action.GetType() != TAction::RESTART_SERVICES) { - return services; - } - - for (const auto &service : action.GetServices()) { - EService value; - if (TryFromString(service, value)) { - services |= value; - } - } - - return services; -} - +static TServices MakeServices(const NKikimrCms::TAction &action) { + TServices services; + + if (action.GetType() != TAction::RESTART_SERVICES) { + return services; + } + + for (const auto &service : action.GetServices()) { + EService value; + if (TryFromString(service, value)) { + services |= value; + } + } + + return services; +} + TSet<TLockableItem *> TClusterInfo::FindLockedItems(const NKikimrCms::TAction &action, const TActorContext *ctx) { @@ -601,20 +601,20 @@ TSet<TLockableItem *> TClusterInfo::FindLockedItems(const NKikimrCms::TAction &a return res; } - switch (action.GetType()) { - case TAction::RESTART_SERVICES: - case TAction::SHUTDOWN_HOST: - if (auto nodes = NodePtrs(action.GetHost(), MakeServices(action))) { - for (const auto node : nodes) { - res.insert(node); - } - } else if (ctx) { + switch (action.GetType()) { + case TAction::RESTART_SERVICES: + case TAction::SHUTDOWN_HOST: + if (auto nodes = NodePtrs(action.GetHost(), MakeServices(action))) { + for (const auto node : nodes) { + res.insert(node); + } + } else if (ctx) { LOG_ERROR_S(*ctx, NKikimrServices::CMS, "FindLockedItems: unknown host " << action.GetHost()); - } - break; - - case TAction::REPLACE_DEVICES: + } + break; + + case TAction::REPLACE_DEVICES: for (const auto &device : action.GetDevices()) { TLockableItem *item = nullptr; @@ -628,14 +628,14 @@ TSet<TLockableItem *> TClusterInfo::FindLockedItems(const NKikimrCms::TAction &a else if (ctx) LOG_ERROR(*ctx, NKikimrServices::CMS, "FindLockedItems: unknown device %s", device.data()); } - break; - - default: - if (ctx) { + break; + + default: + if (ctx) { LOG_ERROR(*ctx, NKikimrServices::CMS, "FindLockedItems: action %s is not supported", TAction::EType_Name(action.GetType()).data()); - } - break; + } + break; } return res; @@ -665,8 +665,8 @@ ui64 TClusterInfo::AddLocks(const TPermissionInfo &permission, const TActorConte if (lock) { if (ctx) - LOG_INFO(*ctx, NKikimrServices::CMS, "Adding lock for %s (permission %s until %s)", - item->PrettyItemName().data(), permission.PermissionId.data(), + LOG_INFO(*ctx, NKikimrServices::CMS, "Adding lock for %s (permission %s until %s)", + item->PrettyItemName().data(), permission.PermissionId.data(), permission.Deadline.ToStringLocalUpToSeconds().data()); item->AddLock(permission); ++locks; @@ -684,8 +684,8 @@ ui64 TClusterInfo::AddExternalLocks(const TNotificationInfo ¬ification, const for (auto item : items) { if (ctx) - LOG_INFO(*ctx, NKikimrServices::CMS, "Adding external lock for %s", - item->PrettyItemName().data()); + LOG_INFO(*ctx, NKikimrServices::CMS, "Adding external lock for %s", + item->PrettyItemName().data()); item->AddExternalLock(notification, action); } @@ -710,7 +710,7 @@ void TClusterInfo::ApplyDowntimes(const TDowntimes &downtimes) } } -void TClusterInfo::UpdateDowntimes(TDowntimes &downtimes, const TActorContext &ctx) +void TClusterInfo::UpdateDowntimes(TDowntimes &downtimes, const TActorContext &ctx) { downtimes.CleanupOld(ctx.Now()); @@ -777,12 +777,12 @@ void TClusterInfo::RollbackLocks(ui64 point) RollbackPoint = point - 1; } -void TClusterInfo::MigrateOldInfo(TClusterInfoPtr old) +void TClusterInfo::MigrateOldInfo(TClusterInfoPtr old) { for (auto &entry : LockableItems) { auto it = old->LockableItems.find(entry.first); if (it != old->LockableItems.end()) - entry.second->MigrateOldInfo(*it->second); + entry.second->MigrateOldInfo(*it->second); } } @@ -798,7 +798,7 @@ void TClusterInfo::DebugDump(const TActorContext &ctx) const << " Host: " << node.Host << Endl << " Address: " << node.Address << Endl << " Version: " << node.Version << Endl - << " State: " << EState_Name(node.State) << Endl; + << " State: " << EState_Name(node.State) << Endl; for (auto pd : node.PDisks) ss << " PDisk: " << pd.NodeId << ":" << pd.DiskId << Endl; for (auto &vd : node.VDisks) @@ -813,7 +813,7 @@ void TClusterInfo::DebugDump(const TActorContext &ctx) const ss << "PDisk {" << Endl << " Id: " << pdisk.PDiskId.NodeId << ":" << pdisk.PDiskId.DiskId << Endl << " NodeId: " << pdisk.NodeId << Endl - << " State: " << EState_Name(pdisk.State) << Endl; + << " State: " << EState_Name(pdisk.State) << Endl; pdisk.DebugLocksDump(ss, " "); ss << "}" << Endl; LOG_DEBUG(ctx, NKikimrServices::CMS, ss.Str()); @@ -825,7 +825,7 @@ void TClusterInfo::DebugDump(const TActorContext &ctx) const << " Id: " << vdisk.VDiskId.ToString() << Endl << " NodeId: " << vdisk.NodeId << Endl << " State: " << EState_Name(vdisk.State) << Endl - << " PDisk: " << vdisk.PDiskId.NodeId << ":" << vdisk.PDiskId.DiskId << Endl; + << " PDisk: " << vdisk.PDiskId.NodeId << ":" << vdisk.PDiskId.DiskId << Endl; for (auto id : vdisk.BSGroups) ss << " BSGroup: " << id << Endl; vdisk.DebugLocksDump(ss, " "); diff --git a/ydb/core/cms/cluster_info.h b/ydb/core/cms/cluster_info.h index b50c9c8a349..af5fe24af80 100644 --- a/ydb/core/cms/cluster_info.h +++ b/ydb/core/cms/cluster_info.h @@ -3,7 +3,7 @@ #include "defs.h" #include "config.h" #include "downtime.h" -#include "services.h" +#include "services.h" #include <library/cpp/actors/interconnect/interconnect.h> #include <ydb/core/base/blobstorage.h> @@ -237,10 +237,10 @@ public: } virtual TString ItemName() const = 0; - virtual TString PrettyItemName() const - { - return ItemName(); - } + virtual TString PrettyItemName() const + { + return ItemName(); + } void AddLock(const TPermissionInfo &permission) { @@ -276,7 +276,7 @@ public: void RemoveScheduledLocks(const TString &requestId); // Fill some item info (e.g. Downtime) basing on previous item state. - virtual void MigrateOldInfo(const TLockableItem &old); + virtual void MigrateOldInfo(const TLockableItem &old); void DebugLocksDump(IOutputStream &ss, const TString &prefix = "") const; @@ -309,10 +309,10 @@ public: TString ItemName() const override { - return Sprintf("Host %s:%" PRIu16 " (%" PRIu32 ")", Host.data(), IcPort, NodeId); + return Sprintf("Host %s:%" PRIu16 " (%" PRIu32 ")", Host.data(), IcPort, NodeId); } - void MigrateOldInfo(const TLockableItem &old) override; + void MigrateOldInfo(const TLockableItem &old) override; ui32 NodeId = 0; TString Host; @@ -326,7 +326,7 @@ public: bool HasTenantInfo = false; TString Tenant; TString PreviousTenant; - TServices Services; + TServices Services; }; using TNodeInfoPtr = TIntrusivePtr<TNodeInfo>; @@ -372,16 +372,16 @@ public: TPDiskInfo &operator=(TPDiskInfo &&other) = default; TString ItemName() const override; - TString PrettyItemName() const override; + TString PrettyItemName() const override; TString GetDeviceName() const; static bool IsDeviceName(const TString &name); static TPDiskID NameToId(const TString &name); - void MigrateOldInfo(const TLockableItem &old) override; + void MigrateOldInfo(const TLockableItem &old) override; TPDiskID PDiskId; ui32 NodeId; - TString Host; + TString Host; TString Path; TSet<TVDiskID> VDisks; // SlotIdx -> VDiskID @@ -414,18 +414,18 @@ public: TVDiskInfo &operator=(TVDiskInfo &&other) = default; TString ItemName() const override; - TString PrettyItemName() const override; + TString PrettyItemName() const override; TString GetDeviceName() const; static bool IsDeviceName(const TString &name); static TVDiskID NameToId(const TString &name); - void MigrateOldInfo(const TLockableItem &old) override; + void MigrateOldInfo(const TLockableItem &old) override; TVDiskID VDiskId; TPDiskID PDiskId; - TString Path; + TString Path; ui32 NodeId; - TString Host; + TString Host; ui32 SlotId; TSet<ui32> BSGroups; @@ -514,20 +514,20 @@ public: return nodes; } - TVector<const TNodeInfo *> TenantNodes(const TString &tenant) const - { - TVector<const TNodeInfo *> nodes; - - auto pr = TenantToNodeId.equal_range(tenant); - for (auto it = pr.first; it != pr.second; ++it) { - const ui32 nodeId = it->second; - Y_VERIFY(HasNode(nodeId)); - nodes.push_back(Nodes.find(nodeId)->second.Get()); - } - - return nodes; - } - + TVector<const TNodeInfo *> TenantNodes(const TString &tenant) const + { + TVector<const TNodeInfo *> nodes; + + auto pr = TenantToNodeId.equal_range(tenant); + for (auto it = pr.first; it != pr.second; ++it) { + const ui32 nodeId = it->second; + Y_VERIFY(HasNode(nodeId)); + nodes.push_back(Nodes.find(nodeId)->second.Get()); + } + + return nodes; + } + size_t NodesCount() const { return Nodes.size(); @@ -576,11 +576,11 @@ public: return PDisks.contains(id); } - bool HasPDisk(const TString &hostName, const TString &path) const - { - return !!HostNamePathToPDiskId(hostName, path); - } - + bool HasPDisk(const TString &hostName, const TString &path) const + { + return !!HostNamePathToPDiskId(hostName, path); + } + const TPDiskInfo &PDisk(TPDiskID pdId) const { Y_VERIFY(HasPDisk(pdId)); @@ -593,11 +593,11 @@ public: return PDisk(id); } - const TPDiskInfo &PDisk(const TString &hostName, const TString &path) const - { - return PDisk(HostNamePathToPDiskId(hostName, path)); - } - + const TPDiskInfo &PDisk(const TString &hostName, const TString &path) const + { + return PDisk(HostNamePathToPDiskId(hostName, path)); + } + size_t PDisksCount() const { return PDisks.size(); @@ -671,8 +671,8 @@ public: void SetTimestamp(TInstant timestamp); - void AddNode(const TEvInterconnect::TNodeInfo &info, const TActorContext *ctx); - void SetNodeState(ui32 nodeId, NKikimrCms::EState state, const NKikimrWhiteboard::TSystemStateInfo &info); + void AddNode(const TEvInterconnect::TNodeInfo &info, const TActorContext *ctx); + void SetNodeState(ui32 nodeId, NKikimrCms::EState state, const NKikimrWhiteboard::TSystemStateInfo &info); void ClearNode(ui32 nodeId); void AddTablet(ui32 nodeId, const NKikimrWhiteboard::TTabletStateInfo &info); void AddPDisk(const NKikimrBlobStorage::TBaseConfig::TPDisk &info); @@ -697,7 +697,7 @@ public: ui64 AddExternalLocks(const TNotificationInfo ¬ification, const TActorContext *ctx); void ApplyDowntimes(const TDowntimes &downtimes); - void UpdateDowntimes(TDowntimes &downtimes, const TActorContext &ctx); + void UpdateDowntimes(TDowntimes &downtimes, const TActorContext &ctx); ui64 AddTempLocks(const NKikimrCms::TAction &action, const TActorContext *ctx); ui64 ScheduleActions(const TRequestInfo &request, const TActorContext *ctx); @@ -711,7 +711,7 @@ public: return ++RollbackPoint; } - void MigrateOldInfo(TClusterInfoPtr old); + void MigrateOldInfo(TClusterInfoPtr old); void ApplyInitialNodeTenants(const TActorContext& ctx, const THashMap<ui32, TString>& nodeTenants); void DebugDump(const TActorContext &ctx) const; @@ -721,13 +721,13 @@ public: bool IsOutdated() const { return Outdated; } void SetOutdated(bool val) { Outdated = val; } - static EGroupConfigurationType VDiskConfigurationType(const TVDiskID &vdId) { - return TGroupID(vdId.GroupID).ConfigurationType(); - } - - static bool IsStaticGroupVDisk(const TVDiskID &vdId) { return GroupConfigurationTypeStatic == VDiskConfigurationType(vdId); } - static bool IsDynamicGroupVDisk(const TVDiskID &vdId) { return GroupConfigurationTypeDynamic == VDiskConfigurationType(vdId); } - + static EGroupConfigurationType VDiskConfigurationType(const TVDiskID &vdId) { + return TGroupID(vdId.GroupID).ConfigurationType(); + } + + static bool IsStaticGroupVDisk(const TVDiskID &vdId) { return GroupConfigurationTypeStatic == VDiskConfigurationType(vdId); } + static bool IsDynamicGroupVDisk(const TVDiskID &vdId) { return GroupConfigurationTypeDynamic == VDiskConfigurationType(vdId); } + private: TNodeInfo &NodeRef(ui32 nodeId) const { @@ -735,7 +735,7 @@ private: return *Nodes.find(nodeId)->second; } - TVector<TNodeInfo *> NodePtrs(const TString &hostName, const TServices &filterByServices = {}) + TVector<TNodeInfo *> NodePtrs(const TString &hostName, const TServices &filterByServices = {}) { TVector<TNodeInfo *> nodes; @@ -746,18 +746,18 @@ private: return nodes; } - auto range = HostNameToNodeId.equal_range(hostName); - for (auto it = range.first; it != range.second; ++it) { + auto range = HostNameToNodeId.equal_range(hostName); + for (auto it = range.first; it != range.second; ++it) { nodeId = it->second; - + Y_VERIFY(HasNode(nodeId)); - auto &node = NodeRef(nodeId); - - if (filterByServices && !(node.Services & filterByServices)) { - continue; - } - - nodes.push_back(&node); + auto &node = NodeRef(nodeId); + + if (filterByServices && !(node.Services & filterByServices)) { + continue; + } + + nodes.push_back(&node); } return nodes; @@ -793,22 +793,22 @@ private: return BSGroups.find(groupId)->second; } - TPDiskID HostNamePathToPDiskId(const TString &hostName, const TString &path) const { - auto pr = HostNameToNodeId.equal_range(hostName); - for (auto it = pr.first; it != pr.second; ++it) { - const ui32 nodeId = it->second; - Y_VERIFY(HasNode(nodeId)); - const auto &node = Node(nodeId); - for (const auto &id : node.PDisks) { - Y_VERIFY(HasPDisk(id)); - if (PDisk(id).Path == path) { - return id; - } - } - } - return TPDiskID(); - } - + TPDiskID HostNamePathToPDiskId(const TString &hostName, const TString &path) const { + auto pr = HostNameToNodeId.equal_range(hostName); + for (auto it = pr.first; it != pr.second; ++it) { + const ui32 nodeId = it->second; + Y_VERIFY(HasNode(nodeId)); + const auto &node = Node(nodeId); + for (const auto &id : node.PDisks) { + Y_VERIFY(HasPDisk(id)); + if (PDisk(id).Path == path) { + return id; + } + } + } + return TPDiskID(); + } + TSet<TLockableItem *> FindLockedItems(const NKikimrCms::TAction &action, const TActorContext *ctx); TNodes Nodes; @@ -823,7 +823,7 @@ private: // Fast access structures. TMultiMap<TString, ui32> HostNameToNodeId; - TMultiMap<TString, ui32> TenantToNodeId; + TMultiMap<TString, ui32> TenantToNodeId; THashMap<TString, TLockableItemPtr> LockableItems; }; diff --git a/ydb/core/cms/cluster_info_ut.cpp b/ydb/core/cms/cluster_info_ut.cpp index 08715c12b46..8b9d3b3d653 100644 --- a/ydb/core/cms/cluster_info_ut.cpp +++ b/ydb/core/cms/cluster_info_ut.cpp @@ -264,7 +264,7 @@ Y_UNIT_TEST_SUITE(TClusterInfoTest) { { 3, "::2", "localhost", "localhost", 1, TNodeLocation() }; TClusterInfo cluster; - cluster.AddNode(node1, nullptr); + cluster.AddNode(node1, nullptr); UNIT_ASSERT(cluster.HasNode(1)); UNIT_ASSERT(!cluster.HasNode(2)); UNIT_ASSERT(cluster.HasNode("test1")); @@ -272,14 +272,14 @@ Y_UNIT_TEST_SUITE(TClusterInfoTest) { UNIT_ASSERT_VALUES_EQUAL(cluster.HostNodes("test1").size(), 1); CheckNode(cluster.Node(1), 1, "test1", "::1", EState::UNKNOWN); - cluster.AddNode(node2, nullptr); + cluster.AddNode(node2, nullptr); UNIT_ASSERT(cluster.HasNode(1)); UNIT_ASSERT(!cluster.HasNode("test1")); UNIT_ASSERT(cluster.HasNode("test2")); UNIT_ASSERT_VALUES_EQUAL(cluster.HostNodes("test2").size(), 1); CheckNode(cluster.Node(1), 1, "test2", "::1", EState::UNKNOWN); - cluster.AddNode(node3, nullptr); + cluster.AddNode(node3, nullptr); UNIT_ASSERT(cluster.HasNode(2)); UNIT_ASSERT(cluster.HasNode("localhost")); UNIT_ASSERT_VALUES_EQUAL(cluster.NodesCount("localhost"), 1); @@ -287,7 +287,7 @@ Y_UNIT_TEST_SUITE(TClusterInfoTest) { UNIT_ASSERT_VALUES_EQUAL(cluster.HostNodes("localhost")[0]->NodeId, 2); CheckNode(cluster.Node(2), 2, "localhost", "::2", EState::UNKNOWN); - cluster.AddNode(node4, nullptr); + cluster.AddNode(node4, nullptr); UNIT_ASSERT(cluster.HasNode(3)); UNIT_ASSERT_VALUES_EQUAL(cluster.NodesCount("localhost"), 2); @@ -374,7 +374,7 @@ Y_UNIT_TEST_SUITE(TClusterInfoTest) { permission.Action.SetHost("2"); permission.Deadline = now - TDuration::Seconds(30); - cluster.SetNodeState(2, DOWN, MakeSystemStateInfo("1")); + cluster.SetNodeState(2, DOWN, MakeSystemStateInfo("1")); UNIT_ASSERT_VALUES_EQUAL(cluster.AddLocks(permission, nullptr), 1); UNIT_ASSERT_VALUES_EQUAL(cluster.Node(2).State, EState::RESTART); UNIT_ASSERT_VALUES_EQUAL(cluster.Node(2).Lock->ActionDeadline, now + TDuration::Seconds(30)); diff --git a/ydb/core/cms/cms.cpp b/ydb/core/cms/cms.cpp index 93c8dd2f157..0a205024e3c 100644 --- a/ydb/core/cms/cms.cpp +++ b/ydb/core/cms/cms.cpp @@ -1,7 +1,7 @@ #include "cms_impl.h" #include "info_collector.h" #include "scheme.h" -#include "sentinel.h" +#include "sentinel.h" #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/appdata.h> @@ -11,7 +11,7 @@ #include <library/cpp/actors/interconnect/interconnect.h> -#include <util/generic/serialized_enum.h> +#include <util/generic/serialized_enum.h> #include <util/string/join.h> #include <util/system/hostname.h> @@ -226,10 +226,10 @@ bool TCms::CheckPermissionRequest(const TPermissionRequest &request, scheduled.SetTenantPolicy(request.GetTenantPolicy()); } - LOG_INFO_S(ctx, NKikimrServices::CMS, + LOG_INFO_S(ctx, NKikimrServices::CMS, "Check request: " << request.ShortDebugString()); - switch (request.GetAvailabilityMode()) { + switch (request.GetAvailabilityMode()) { case MODE_MAX_AVAILABILITY: case MODE_KEEP_AVAILABLE: case MODE_FORCE_RESTART: @@ -238,22 +238,22 @@ bool TCms::CheckPermissionRequest(const TPermissionRequest &request, response.MutableStatus()->SetCode(TStatus::WRONG_REQUEST); response.MutableStatus() ->SetReason(Sprintf("Unsupported availability mode: %s", - EAvailabilityMode_Name(request.GetAvailabilityMode()).data())); + EAvailabilityMode_Name(request.GetAvailabilityMode()).data())); return false; }; auto point = ClusterInfo->PushRollbackPoint(); for (const auto &action : request.GetActions()) { - TDuration permissionDuration = State->Config.DefaultPermissionDuration; - if (request.HasDuration()) - permissionDuration = TDuration::MicroSeconds(request.GetDuration()); - if (action.HasDuration()) - permissionDuration = TDuration::MicroSeconds(action.GetDuration()); - - TActionOptions opts(permissionDuration); - opts.TenantPolicy = request.GetTenantPolicy(); - opts.AvailabilityMode = request.GetAvailabilityMode(); - + TDuration permissionDuration = State->Config.DefaultPermissionDuration; + if (request.HasDuration()) + permissionDuration = TDuration::MicroSeconds(request.GetDuration()); + if (action.HasDuration()) + permissionDuration = TDuration::MicroSeconds(action.GetDuration()); + + TActionOptions opts(permissionDuration); + opts.TenantPolicy = request.GetTenantPolicy(); + opts.AvailabilityMode = request.GetAvailabilityMode(); + TErrorInfo error; LOG_DEBUG(ctx, NKikimrServices::CMS, "Checking action: %s", action.ShortDebugString().data()); @@ -342,17 +342,17 @@ bool TCms::IsActionHostValid(const TAction &action, TErrorInfo &error) const return true; } -bool TCms::ParseServices(const TAction &action, TServices &services, TErrorInfo &error) const { +bool TCms::ParseServices(const TAction &action, TServices &services, TErrorInfo &error) const { for (const auto &service : action.GetServices()) { - EService value; - if (!TryFromString(service, value)) { + EService value; + if (!TryFromString(service, value)) { error.Code = TStatus::WRONG_REQUEST; - error.Reason = Sprintf("Invalid service '%s' (supported services: %s)", - service.data(), GetEnumAllNames<EService>().data()); + error.Reason = Sprintf("Invalid service '%s' (supported services: %s)", + service.data(), GetEnumAllNames<EService>().data()); return false; } - - services |= value; + + services |= value; } return true; @@ -439,22 +439,22 @@ bool TCms::CheckAction(const TAction &action, } } -bool TCms::CheckActionShutdownNode(const NKikimrCms::TAction &action, - const TActionOptions &opts, - const TNodeInfo &node, - TErrorInfo &error) const -{ - if (!TryToLockNode(action, opts, node, error)) { - return false; - } - - if (!TryToLockVDisks(action, opts, node.VDisks, error)) { - return false; - } - - return true; -} - +bool TCms::CheckActionShutdownNode(const NKikimrCms::TAction &action, + const TActionOptions &opts, + const TNodeInfo &node, + TErrorInfo &error) const +{ + if (!TryToLockNode(action, opts, node, error)) { + return false; + } + + if (!TryToLockVDisks(action, opts, node.VDisks, error)) { + return false; + } + + return true; +} + bool TCms::CheckActionRestartServices(const TAction &action, const TActionOptions &opts, TErrorInfo &error) const @@ -469,35 +469,35 @@ bool TCms::CheckActionRestartServices(const TAction &action, return false; } - bool found = false; - for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { - if (node->Services & services) { - found = true; - if (!CheckActionShutdownNode(action, opts, *node, error)) { - return false; - } - } - } - + bool found = false; + for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { + if (node->Services & services) { + found = true; + if (!CheckActionShutdownNode(action, opts, *node, error)) { + return false; + } + } + } + if (!found) { - error.Code = TStatus::NO_SUCH_SERVICE; - error.Reason = Sprintf("No such services: %s on host %s", - JoinSeq(", ", action.GetServices()).c_str(), action.GetHost().c_str()); - return false; - } - - error.Deadline = TActivationContext::Now() + opts.PermissionDuration; - return true; + error.Code = TStatus::NO_SUCH_SERVICE; + error.Reason = Sprintf("No such services: %s on host %s", + JoinSeq(", ", action.GetServices()).c_str(), action.GetHost().c_str()); + return false; + } + + error.Deadline = TActivationContext::Now() + opts.PermissionDuration; + return true; } bool TCms::CheckActionShutdownHost(const TAction &action, const TActionOptions &opts, TErrorInfo &error) const { - for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { - if (!CheckActionShutdownNode(action, opts, *node, error)) { + for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { + if (!CheckActionShutdownNode(action, opts, *node, error)) { return false; - } + } } error.Deadline = TActivationContext::Now() + opts.PermissionDuration; @@ -667,7 +667,7 @@ bool TCms::TryToLockVDisk(const TActionOptions& opts, } // Check if group already has locked disks. - if (counters.Locked && counters.Code == TStatus::DISALLOW) { + if (counters.Locked && counters.Code == TStatus::DISALLOW) { error.Code = counters.Code; error.Reason = JoinSeq("\n", counters.Errors); error.Deadline = counters.Deadline; @@ -676,7 +676,7 @@ bool TCms::TryToLockVDisk(const TActionOptions& opts, switch (opts.AvailabilityMode) { case MODE_MAX_AVAILABILITY: - if ((counters.Down + counters.Locked) > 0) { + if ((counters.Down + counters.Locked) > 0) { Y_VERIFY(counters.Code == TStatus::DISALLOW_TEMP); error.Code = TStatus::DISALLOW_TEMP; error.Reason = JoinSeq("\n", counters.Errors); @@ -685,7 +685,7 @@ bool TCms::TryToLockVDisk(const TActionOptions& opts, } break; case MODE_KEEP_AVAILABLE: - if ((counters.Down + counters.Locked) >= group.Erasure.ParityParts()) { + if ((counters.Down + counters.Locked) >= group.Erasure.ParityParts()) { Y_VERIFY(counters.Code == TStatus::DISALLOW_TEMP); error.Code = TStatus::DISALLOW_TEMP; error.Reason = JoinSeq("\n", counters.Errors); @@ -727,14 +727,14 @@ bool TCms::CheckActionReplaceDevices(const TAction &action, res = false; break; } - } else if (ClusterInfo->HasPDisk(action.GetHost(), device)) { - const auto &pdisk = ClusterInfo->PDisk(action.GetHost(), device); - if (TryToLockPDisk(action, opts, pdisk, error)) - ClusterInfo->AddPDiskTempLock(pdisk.PDiskId, action); - else { - res = false; - break; - } + } else if (ClusterInfo->HasPDisk(action.GetHost(), device)) { + const auto &pdisk = ClusterInfo->PDisk(action.GetHost(), device); + if (TryToLockPDisk(action, opts, pdisk, error)) + ClusterInfo->AddPDiskTempLock(pdisk.PDiskId, action); + else { + res = false; + break; + } } else if (ClusterInfo->HasVDisk(device)) { const auto &vdisk = ClusterInfo->VDisk(device); if (TryToLockVDisk(opts, vdisk, duration, error)) @@ -759,40 +759,40 @@ bool TCms::CheckActionReplaceDevices(const TAction &action, } void TCms::AcceptPermissions(TPermissionResponse &resp, const TString &requestId, - const TString &owner, const TActorContext &ctx, bool check) + const TString &owner, const TActorContext &ctx, bool check) { for (size_t i = 0; i < resp.PermissionsSize(); ++i) { auto &permission = *resp.MutablePermissions(i); permission.SetId(owner + "-p-" + ToString(State->NextPermissionId++)); State->Permissions.emplace(permission.GetId(), TPermissionInfo(permission, requestId, owner)); ClusterInfo->AddLocks(permission, requestId, owner, &ctx); - - if (!check || owner != WALLE_CMS_USER) { - continue; - } - - auto reqIt = State->WalleRequests.find(requestId); - if (reqIt == State->WalleRequests.end()) { - LOG_ERROR_S(ctx, NKikimrServices::CMS, "Cannot add permission to unknown wall-e request " << requestId); - continue; - } - - auto taskIt = State->WalleTasks.find(reqIt->second); - if (taskIt == State->WalleTasks.end()) { - LOG_ERROR_S(ctx, NKikimrServices::CMS, "Cannot add permission to unknown wall-e task" << reqIt->second); - continue; - } - - taskIt->second.Permissions.insert(permission.GetId()); - } -} - -void TCms::ScheduleUpdateClusterInfo(const TActorContext &ctx, bool now) -{ - ctx.Schedule(now ? TDuration::Zero() : TDuration::Minutes(1), - new TEvPrivate::TEvUpdateClusterInfo()); -} - + + if (!check || owner != WALLE_CMS_USER) { + continue; + } + + auto reqIt = State->WalleRequests.find(requestId); + if (reqIt == State->WalleRequests.end()) { + LOG_ERROR_S(ctx, NKikimrServices::CMS, "Cannot add permission to unknown wall-e request " << requestId); + continue; + } + + auto taskIt = State->WalleTasks.find(reqIt->second); + if (taskIt == State->WalleTasks.end()) { + LOG_ERROR_S(ctx, NKikimrServices::CMS, "Cannot add permission to unknown wall-e task" << reqIt->second); + continue; + } + + taskIt->second.Permissions.insert(permission.GetId()); + } +} + +void TCms::ScheduleUpdateClusterInfo(const TActorContext &ctx, bool now) +{ + ctx.Schedule(now ? TDuration::Zero() : TDuration::Minutes(1), + new TEvPrivate::TEvUpdateClusterInfo()); +} + void TCms::ScheduleCleanup(TInstant time, const TActorContext &ctx) { // Don't schedule event in the past or earlier then already @@ -817,10 +817,10 @@ void TCms::SchedulePermissionsCleanup(const TActorContext &ctx) TInstant earliest = TInstant::Max(); for (const auto &entry : State->Permissions) { - const TDuration duration = TDuration::MicroSeconds(entry.second.Action.GetDuration()); - const TDuration doubleDuration = ((TDuration::Max() / 2) >= duration ? (2 * duration) : TDuration::Max()); - const TInstant deadline = entry.second.Deadline; - earliest = Min(earliest, deadline + doubleDuration); + const TDuration duration = TDuration::MicroSeconds(entry.second.Action.GetDuration()); + const TDuration doubleDuration = ((TDuration::Max() / 2) >= duration ? (2 * duration) : TDuration::Max()); + const TInstant deadline = entry.second.Deadline; + earliest = Min(earliest, deadline + doubleDuration); } ScheduleCleanup(earliest, ctx); @@ -866,7 +866,7 @@ void TCms::CleanupExpired(const TActorContext &ctx) SchedulePermissionsCleanup(ctx); ScheduleNotificationsCleanup(ctx); } - + void TCms::DoPermissionsCleanup(const TActorContext &ctx) { ScheduledCleanups.pop(); @@ -874,14 +874,14 @@ void TCms::DoPermissionsCleanup(const TActorContext &ctx) TVector<TString> ids; auto now = ctx.Now(); for (const auto &entry : State->Permissions) { - const TDuration duration = TDuration::MicroSeconds(entry.second.Action.GetDuration()); - const TDuration doubleDuration = ((TDuration::Max() / 2) >= duration ? (2 * duration) : TDuration::Max()); - const TInstant deadline(entry.second.Deadline); - if ((deadline + doubleDuration) <= now) + const TDuration duration = TDuration::MicroSeconds(entry.second.Action.GetDuration()); + const TDuration doubleDuration = ((TDuration::Max() / 2) >= duration ? (2 * duration) : TDuration::Max()); + const TInstant deadline(entry.second.Deadline); + if ((deadline + doubleDuration) <= now) ids.push_back(entry.first); } - Execute(CreateTxRemovePermissions(std::move(ids), nullptr, nullptr, true), ctx); + Execute(CreateTxRemovePermissions(std::move(ids), nullptr, nullptr, true), ctx); } void TCms::CleanupWalleTasks(const TActorContext &ctx) @@ -892,28 +892,28 @@ void TCms::CleanupWalleTasks(const TActorContext &ctx) // permissions which means we might have some Wall-E requests // not attached to Wall-E tasks and Wall-E tasks with no // request and permissions. Cleanup the mess here. - TVector<TString> requestsToRemove; + TVector<TString> requestsToRemove; for (const auto &entry : State->ScheduledRequests) { const auto &request = entry.second; if (request.Owner == WALLE_CMS_USER && !State->WalleRequests.contains(request.RequestId)) - requestsToRemove.push_back(request.RequestId); + requestsToRemove.push_back(request.RequestId); + } + + for (const auto &requestId : requestsToRemove) { + Execute(CreateTxRemoveRequest(requestId, nullptr, nullptr), ctx); } - for (const auto &requestId : requestsToRemove) { - Execute(CreateTxRemoveRequest(requestId, nullptr, nullptr), ctx); - } - - TVector<TString> permissionsToRemove; + TVector<TString> permissionsToRemove; for (const auto &entry : State->Permissions) { const auto &permission = entry.second; if (permission.Owner == WALLE_CMS_USER && !State->WalleRequests.contains(permission.RequestId)) - permissionsToRemove.push_back(permission.PermissionId); + permissionsToRemove.push_back(permission.PermissionId); } - if (!permissionsToRemove.empty()) - Execute(CreateTxRemovePermissions(permissionsToRemove, nullptr, nullptr), ctx); + if (!permissionsToRemove.empty()) + Execute(CreateTxRemovePermissions(permissionsToRemove, nullptr, nullptr), ctx); RemoveEmptyWalleTasks(ctx); @@ -943,8 +943,8 @@ void TCms::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::CMS, "TCms::Cleanup"); - if (State->Sentinel) - ctx.Send(State->Sentinel, new TEvents::TEvPoisonPill); + if (State->Sentinel) + ctx.Send(State->Sentinel, new TEvents::TEvPoisonPill); } void TCms::Die(const TActorContext& ctx) @@ -962,18 +962,18 @@ void TCms::AddHostState(const TNodeInfo &node, TClusterStateResponse &resp, TIns host->SetInterconnectPort(node.IcPort); host->SetTimestamp(timestamp.GetValue()); if (node.State == UP || node.VDisks || node.PDisks) { - for (const auto flag : GetEnumAllValues<EService>()) { - if (!(node.Services & flag)) { - continue; - } - - auto* service = host->AddServices(); - service->SetName(ToString(flag)); - service->SetState(node.State); - if (node.State == UP) { - service->SetVersion(node.Version); - } - service->SetTimestamp(timestamp.GetValue()); + for (const auto flag : GetEnumAllValues<EService>()) { + if (!(node.Services & flag)) { + continue; + } + + auto* service = host->AddServices(); + service->SetName(ToString(flag)); + service->SetState(node.State); + if (node.State == UP) { + service->SetVersion(node.Version); + } + service->SetTimestamp(timestamp.GetValue()); } for (const auto &vdId : node.VDisks) { @@ -1000,7 +1000,7 @@ void TCms::GetPermission(TEvCms::TEvManagePermissionRequest::TPtr &ev, bool all, const auto &rec = ev->Get()->Record; const TString &user = rec.GetUser(); - LOG_INFO(ctx, NKikimrServices::CMS, "Get %s permissions for %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Get %s permissions for %s", all ? "all" : "selected", user.data()); resp->Record.MutableStatus()->SetCode(TStatus::OK); @@ -1044,7 +1044,7 @@ void TCms::RemovePermission(TEvCms::TEvManagePermissionRequest::TPtr &ev, bool d const auto &rec = ev->Get()->Record; const TString &user = rec.GetUser(); - LOG_INFO(ctx, NKikimrServices::CMS, "User %s %s permissions %s", + LOG_INFO(ctx, NKikimrServices::CMS, "User %s %s permissions %s", user.data(), done ? "is done with" : "rejected", ToString(rec.GetPermissions()).data()); TVector<TString> ids; @@ -1071,8 +1071,8 @@ void TCms::RemovePermission(TEvCms::TEvManagePermissionRequest::TPtr &ev, bool d LOG_DEBUG(ctx, NKikimrServices::CMS, "Resulting status: %s %s", TStatus::ECode_Name(resp->Record.GetStatus().GetCode()).data(), resp->Record.GetStatus().GetReason().data()); - if (!rec.GetDryRun() && resp->Record.GetStatus().GetCode() == TStatus::OK) { - auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); + if (!rec.GetDryRun() && resp->Record.GetStatus().GetCode() == TStatus::OK) { + auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); Execute(CreateTxRemovePermissions(std::move(ids), std::move(ev->Release()), handle), ctx); } else { Reply(ev, std::move(resp), ctx); @@ -1085,7 +1085,7 @@ void TCms::GetRequest(TEvCms::TEvManageRequestRequest::TPtr &ev, bool all, const const auto &rec = ev->Get()->Record; const TString &user = rec.GetUser(); - LOG_INFO(ctx, NKikimrServices::CMS, "Get %s requests for %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Get %s requests for %s", all ? "all" : "selected", user.data()); resp->Record.MutableStatus()->SetCode(TStatus::OK); @@ -1124,7 +1124,7 @@ void TCms::RemoveRequest(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActor const TString &user = rec.GetUser(); const TString &id = rec.GetRequestId(); - LOG_INFO(ctx, NKikimrServices::CMS, "User %s removes request %s", user.data(), id.data()); + LOG_INFO(ctx, NKikimrServices::CMS, "User %s removes request %s", user.data(), id.data()); resp->Record.MutableStatus()->SetCode(TStatus::OK); auto it = State->ScheduledRequests.find(id); @@ -1143,8 +1143,8 @@ void TCms::RemoveRequest(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActor LOG_DEBUG(ctx, NKikimrServices::CMS, "Resulting status: %s %s", TStatus::ECode_Name(resp->Record.GetStatus().GetCode()).data(), resp->Record.GetStatus().GetReason().data()); - if (!rec.GetDryRun() && resp->Record.GetStatus().GetCode() == TStatus::OK) { - auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); + if (!rec.GetDryRun() && resp->Record.GetStatus().GetCode() == TStatus::OK) { + auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); Execute(CreateTxRemoveRequest(id, std::move(ev->Release()), handle), ctx); } else { Reply(ev, std::move(resp), ctx); @@ -1159,7 +1159,7 @@ void TCms::GetNotifications(TEvCms::TEvManageNotificationRequest::TPtr &ev, bool const auto &rec = ev->Get()->Record; const TString &user = rec.GetUser(); - LOG_INFO(ctx, NKikimrServices::CMS, "Get %s notifications for %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Get %s notifications for %s", all ? "all" : "selected", user.data()); resp->Record.MutableStatus()->SetCode(TStatus::OK); @@ -1228,8 +1228,8 @@ void TCms::CheckAndEnqueueRequest(TEvCms::TEvPermissionRequest::TPtr &ev, const auto &rec = ev->Get()->Record; if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvPermissionResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } EnqueueRequest(ev.Release(), ctx); @@ -1240,19 +1240,19 @@ void TCms::CheckAndEnqueueRequest(TEvCms::TEvCheckRequest::TPtr &ev, const TActo auto &rec = ev->Get()->Record; if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvPermissionResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } auto it = State->ScheduledRequests.find(rec.GetRequestId()); if (it == State->ScheduledRequests.end()) { auto reason = Sprintf("Unknown request %s", rec.GetRequestId().data()); - return ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::WRONG_REQUEST, reason, ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::WRONG_REQUEST, reason, ctx); } if (it->second.Owner != rec.GetUser()) { auto reason = Sprintf("Request %s doesn't belong to %s", rec.GetRequestId().data(), rec.GetUser().data()); - return ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::WRONG_REQUEST, reason, ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::WRONG_REQUEST, reason, ctx); } EnqueueRequest(ev.Release(), ctx); @@ -1260,7 +1260,7 @@ void TCms::CheckAndEnqueueRequest(TEvCms::TEvCheckRequest::TPtr &ev, const TActo void TCms::CheckAndEnqueueRequest(TEvCms::TEvConditionalPermissionRequest::TPtr &ev, const TActorContext &ctx) { - ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::ERROR, "Not supported", ctx); + ReplyWithError<TEvCms::TEvPermissionResponse>(ev, TStatus::ERROR, "Not supported", ctx); } void TCms::CheckAndEnqueueRequest(TEvCms::TEvNotification::TPtr &ev, const TActorContext &ctx) @@ -1268,8 +1268,8 @@ void TCms::CheckAndEnqueueRequest(TEvCms::TEvNotification::TPtr &ev, const TActo auto &rec = ev->Get()->Record; if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvNotificationResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvNotificationResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } EnqueueRequest(ev.Release(), ctx); @@ -1286,7 +1286,7 @@ void TCms::PersistNodeTenants(TTransactionContext& txc, const TActorContext& ctx auto row = db.Table<Schema::NodeTenant>().Key(nodeId); row.Update(NIceDb::TUpdate<Schema::NodeTenant::Tenant>(tenant)); - LOG_NOTICE(ctx, NKikimrServices::CMS, + LOG_NOTICE(ctx, NKikimrServices::CMS, "Persist node %" PRIu32 " tenant '%s'", nodeId, tenant.data()); } @@ -1304,7 +1304,7 @@ void TCms::ProcessRequest(TAutoPtr<IEventHandle> &ev, const TActorContext &ctx) { TRACE_EVENT(NKikimrServices::CMS); switch (ev->GetTypeRewrite()) { - HFuncTraced(TEvPrivate::TEvUpdateClusterInfo, Handle); + HFuncTraced(TEvPrivate::TEvUpdateClusterInfo, Handle); HFuncTraced(TEvCms::TEvClusterStateRequest, Handle); HFuncTraced(TEvCms::TEvPermissionRequest, Handle); HFuncTraced(TEvCms::TEvCheckRequest, Handle); @@ -1319,15 +1319,15 @@ void TCms::ProcessRequest(TAutoPtr<IEventHandle> &ev, const TActorContext &ctx) void TCms::OnBSCPipeDestroyed(const TActorContext &ctx) { - LOG_WARN(ctx, NKikimrServices::CMS, "BS Controller connection error"); + LOG_WARN(ctx, NKikimrServices::CMS, "BS Controller connection error"); if (State->BSControllerPipe) { NTabletPipe::CloseClient(ctx, State->BSControllerPipe); State->BSControllerPipe = TActorId(); } - if (State->Sentinel) - ctx.Send(State->Sentinel, new TEvSentinel::TEvBSCPipeDisconnected); + if (State->Sentinel) + ctx.Send(State->Sentinel, new TEvSentinel::TEvBSCPipeDisconnected); } void TCms::Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx) @@ -1350,7 +1350,7 @@ void TCms::Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx info->SetOutdated(false); if (ClusterInfo) { - info->MigrateOldInfo(ClusterInfo); + info->MigrateOldInfo(ClusterInfo); } else { info->ApplyDowntimes(State->Downtimes); } @@ -1360,7 +1360,7 @@ void TCms::Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx State->ClusterInfo = info; ClusterInfo = info; - ClusterInfo->UpdateDowntimes(State->Downtimes, ctx); + ClusterInfo->UpdateDowntimes(State->Downtimes, ctx); Execute(CreateTxUpdateDowntimes(), ctx); if (State->InitialNodeTenants) { @@ -1368,9 +1368,9 @@ void TCms::Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx State->InitialNodeTenants.clear(); } - if (State->Config.SentinelConfig.Enable && !State->Sentinel) + if (State->Config.SentinelConfig.Enable && !State->Sentinel) State->Sentinel = RegisterWithSameMailbox(CreateSentinel(State)); - + info->DebugDump(ctx); ProcessQueue(ctx); @@ -1381,20 +1381,20 @@ void TCms::Handle(TEvPrivate::TEvLogAndSend::TPtr &ev, const TActorContext &ctx) Execute(CreateTxLogAndSend(ev), ctx); } -void TCms::Handle(TEvPrivate::TEvUpdateClusterInfo::TPtr &/*ev*/, const TActorContext &ctx) -{ - if (State->ClusterInfo->IsOutdated()) { - ScheduleUpdateClusterInfo(ctx); - } -} - +void TCms::Handle(TEvPrivate::TEvUpdateClusterInfo::TPtr &/*ev*/, const TActorContext &ctx) +{ + if (State->ClusterInfo->IsOutdated()) { + ScheduleUpdateClusterInfo(ctx); + } +} + void TCms::Handle(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActorContext &ctx) { auto &rec = ev->Get()->Record; if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvManageRequestResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvManageRequestResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } switch (rec.GetCommand()) { @@ -1408,8 +1408,8 @@ void TCms::Handle(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActorContext RemoveRequest(ev, ctx); return; default: - return ReplyWithError<TEvCms::TEvManageRequestResponse>( - ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); + return ReplyWithError<TEvCms::TEvManageRequestResponse>( + ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); } } @@ -1418,8 +1418,8 @@ void TCms::Handle(TEvCms::TEvManagePermissionRequest::TPtr &ev, const TActorCont auto &rec = ev->Get()->Record; if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvManagePermissionResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvManagePermissionResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } switch (rec.GetCommand()) { @@ -1435,12 +1435,12 @@ void TCms::Handle(TEvCms::TEvManagePermissionRequest::TPtr &ev, const TActorCont case TManagePermissionRequest::REJECT: RemovePermission(ev, false, ctx); return; - case TManagePermissionRequest::EXTEND: - return ReplyWithError<TEvCms::TEvManagePermissionResponse>( - ev, TStatus::ERROR, "Not supported", ctx); + case TManagePermissionRequest::EXTEND: + return ReplyWithError<TEvCms::TEvManagePermissionResponse>( + ev, TStatus::ERROR, "Not supported", ctx); default: - return ReplyWithError<TEvCms::TEvManagePermissionResponse>( - ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); + return ReplyWithError<TEvCms::TEvManagePermissionResponse>( + ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); } } @@ -1448,8 +1448,8 @@ void TCms::Handle(TEvCms::TEvClusterStateRequest::TPtr &ev, const TActorContext &ctx) { if (ClusterInfo->IsOutdated()) { - return ReplyWithError<TEvCms::TEvClusterStateResponse>( - ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); + return ReplyWithError<TEvCms::TEvClusterStateResponse>( + ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); } TAutoPtr<TEvCms::TEvClusterStateResponse> resp = new TEvCms::TEvClusterStateResponse; @@ -1462,7 +1462,7 @@ void TCms::Handle(TEvCms::TEvClusterStateRequest::TPtr &ev, AddHostState(*node, resp->Record, ClusterInfo->GetTimestamp()); } } else { - return ReplyWithError<TEvCms::TEvClusterStateResponse>( + return ReplyWithError<TEvCms::TEvClusterStateResponse>( ev, TStatus::NO_SUCH_HOST, "Unknown host " + host, ctx); } } @@ -1481,8 +1481,8 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, const TActorContext &ctx) { if (ClusterInfo->IsOutdated()) { - return ReplyWithError<TEvCms::TEvPermissionResponse>( - ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>( + ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); } TAutoPtr<TEvCms::TEvPermissionResponse> resp = new TEvCms::TEvPermissionResponse; @@ -1490,27 +1490,27 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, auto &rec = ev->Get()->Record; TString user = rec.GetUser(); - auto actions(std::move(*rec.MutableActions())); - rec.ClearActions(); - - THashSet<TString> hostNames; - for (const auto &action : actions) { - if (action.HasTenant()) { - for (const TNodeInfo * node : ClusterInfo->TenantNodes(action.GetTenant())) { - if (!hostNames.insert(node->Host).second) { - continue; - } - - TAction &hostAction = *rec.MutableActions()->Add(); - hostAction.CopyFrom(action); - hostAction.ClearTenant(); - hostAction.SetHost(node->Host); - } - } else { - rec.MutableActions()->Add()->CopyFrom(action); - } - } - + auto actions(std::move(*rec.MutableActions())); + rec.ClearActions(); + + THashSet<TString> hostNames; + for (const auto &action : actions) { + if (action.HasTenant()) { + for (const TNodeInfo * node : ClusterInfo->TenantNodes(action.GetTenant())) { + if (!hostNames.insert(node->Host).second) { + continue; + } + + TAction &hostAction = *rec.MutableActions()->Add(); + hostAction.CopyFrom(action); + hostAction.ClearTenant(); + hostAction.SetHost(node->Host); + } + } else { + rec.MutableActions()->Add()->CopyFrom(action); + } + } + bool ok = CheckPermissionRequest(rec, resp->Record, scheduled.Request, ctx); // Schedule request if required. @@ -1529,17 +1529,17 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, copy = new TRequestInfo(scheduled); State->ScheduledRequests.emplace(reqId, std::move(scheduled)); - } else if (user == WALLE_CMS_USER) { - scheduled.Owner = user; - scheduled.RequestId = reqId; - - copy = new TRequestInfo(scheduled); + } else if (user == WALLE_CMS_USER) { + scheduled.Owner = user; + scheduled.RequestId = reqId; + + copy = new TRequestInfo(scheduled); } if (ok) AcceptPermissions(resp->Record, reqId, user, ctx); - auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); + auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); Execute(CreateTxStorePermissions(std::move(ev->Release()), handle, user, std::move(copy)), ctx); } } @@ -1547,8 +1547,8 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) { if (ClusterInfo->IsOutdated()) { - return ReplyWithError<TEvCms::TEvPermissionResponse>( - ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>( + ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); } auto &rec = ev->Get()->Record; @@ -1558,8 +1558,8 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) // deleted after previous event check. if (it == State->ScheduledRequests.end()) { auto reason = Sprintf("Unknown request %s", rec.GetRequestId().data()); - return ReplyWithError<TEvCms::TEvPermissionResponse>( - ev, TStatus::WRONG_REQUEST, reason, ctx); + return ReplyWithError<TEvCms::TEvPermissionResponse>( + ev, TStatus::WRONG_REQUEST, reason, ctx); } TString user = rec.GetUser(); @@ -1600,9 +1600,9 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) } if (ok) - AcceptPermissions(resp->Record, scheduled.RequestId, user, ctx, true); + AcceptPermissions(resp->Record, scheduled.RequestId, user, ctx, true); - auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); + auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); Execute(CreateTxStorePermissions(std::move(ev->Release()), handle, user, std::move(copy)), ctx); } } @@ -1651,9 +1651,9 @@ bool TCms::CheckNotificationReplaceDevices(const TAction &action, TInstant time, TErrorInfo &error, const TActorContext &ctx) const { for (const auto &device : action.GetDevices()) { - if (!ClusterInfo->HasPDisk(device) - && !ClusterInfo->HasPDisk(action.GetHost(), device) - && !ClusterInfo->HasVDisk(device)) { + if (!ClusterInfo->HasPDisk(device) + && !ClusterInfo->HasPDisk(action.GetHost(), device) + && !ClusterInfo->HasVDisk(device)) { error.Code = TStatus::NO_SUCH_DEVICE; error.Reason = Sprintf("Unknown device %s (use cluster state command" " to get list of known devices)", device.data()); @@ -1739,8 +1739,8 @@ bool TCms::CheckNotification(const TNotification ¬ification, void TCms::Handle(TEvCms::TEvNotification::TPtr &ev, const TActorContext &ctx) { if (ClusterInfo->IsOutdated()) { - return ReplyWithError<TEvCms::TEvNotificationResponse>( - ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); + return ReplyWithError<TEvCms::TEvNotificationResponse>( + ev, TStatus::ERROR_TEMP, "Cannot collect cluster state", ctx); } Execute(CreateTxProcessNotification(ev), ctx); @@ -1750,12 +1750,12 @@ void TCms::Handle(TEvCms::TEvManageNotificationRequest::TPtr &ev, const TActorCo { auto &rec = ev->Get()->Record; - LOG_INFO(ctx, NKikimrServices::CMS, "Notification management request: %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Notification management request: %s", rec.ShortDebugString().data()); if (!rec.GetUser()) { - return ReplyWithError<TEvCms::TEvManageNotificationResponse>( - ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); + return ReplyWithError<TEvCms::TEvManageNotificationResponse>( + ev, TStatus::WRONG_REQUEST, "Missing user in request", ctx); } switch (rec.GetCommand()) { @@ -1769,8 +1769,8 @@ void TCms::Handle(TEvCms::TEvManageNotificationRequest::TPtr &ev, const TActorCo Execute(CreateTxRejectNotification(ev), ctx); return; default: - return ReplyWithError<TEvCms::TEvManageNotificationResponse>( - ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); + return ReplyWithError<TEvCms::TEvManageNotificationResponse>( + ev, TStatus::WRONG_REQUEST, "Unknown command", ctx); } } @@ -1804,7 +1804,7 @@ void TCms::Handle(TEvCms::TEvStoreWalleTask::TPtr &ev, const TActorContext &ctx) State->WalleTasks.emplace(event->Task.TaskId, event->Task); State->WalleRequests.emplace(event->Task.RequestId, event->Task.TaskId); - auto handle = new IEventHandle(ev->Sender, SelfId(), new TEvCms::TEvWalleTaskStored(event->Task.TaskId), 0, ev->Cookie); + auto handle = new IEventHandle(ev->Sender, SelfId(), new TEvCms::TEvWalleTaskStored(event->Task.TaskId), 0, ev->Cookie); Execute(CreateTxStoreWalleTask(event->Task, std::move(ev->Release()), handle), ctx); } @@ -1815,10 +1815,10 @@ void TCms::Handle(TEvCms::TEvRemoveWalleTask::TPtr &ev, const TActorContext &ctx if (State->WalleTasks.contains(id)) { auto &task = State->WalleTasks.find(id)->second; - auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); - if (State->ScheduledRequests.contains(task.RequestId)) { + auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); + if (State->ScheduledRequests.contains(task.RequestId)) { Execute(CreateTxRemoveRequest(task.RequestId, std::move(ev->Release()), handle), ctx); - } else { + } else { TVector<TString> ids(task.Permissions.begin(), task.Permissions.end()); Execute(CreateTxRemovePermissions(ids, std::move(ev->Release()), handle), ctx); } @@ -1844,14 +1844,14 @@ void TCms::Handle(TEvCms::TEvSetConfigRequest::TPtr &ev, const TActorContext &ct void TCms::Handle(TEvCms::TEvResetMarkerRequest::TPtr &ev, const TActorContext &ctx) { - return ReplyWithError<TEvCms::TEvResetMarkerResponse>( - ev, TStatus::ERROR, "Unsupported action", ctx); + return ReplyWithError<TEvCms::TEvResetMarkerResponse>( + ev, TStatus::ERROR, "Unsupported action", ctx); } void TCms::Handle(TEvCms::TEvSetMarkerRequest::TPtr &ev, const TActorContext &ctx) { - return ReplyWithError<TEvCms::TEvSetMarkerResponse>( - ev, TStatus::ERROR, "Unsupported action", ctx); + return ReplyWithError<TEvCms::TEvSetMarkerResponse>( + ev, TStatus::ERROR, "Unsupported action", ctx); } void TCms::Handle(TEvCms::TEvGetLogTailRequest::TPtr &ev, const TActorContext &ctx) diff --git a/ydb/core/cms/cms.h b/ydb/core/cms/cms.h index 39b46b77002..25cb8ef024b 100644 --- a/ydb/core/cms/cms.h +++ b/ydb/core/cms/cms.h @@ -195,7 +195,7 @@ struct TEvCms { TString ToString() const override { - return Sprintf("%s { Task: %s }", ToStringHeader().data(), Task.ToString().data()); + return Sprintf("%s { Task: %s }", ToStringHeader().data(), Task.ToString().data()); } }; @@ -209,7 +209,7 @@ struct TEvCms { TString ToString() const override { - return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); + return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); } }; @@ -218,7 +218,7 @@ struct TEvCms { TString ToString() const override { - return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); + return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); } }; @@ -232,7 +232,7 @@ struct TEvCms { TString ToString() const override { - return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); + return Sprintf("%s { TaskId: %s }", ToStringHeader().data(), TaskId.data()); } }; diff --git a/ydb/core/cms/cms_impl.h b/ydb/core/cms/cms_impl.h index 718b0d3e1d1..48138884907 100644 --- a/ydb/core/cms/cms_impl.h +++ b/ydb/core/cms/cms_impl.h @@ -1,10 +1,10 @@ #pragma once -#include "audit_log.h" +#include "audit_log.h" #include "cms.h" #include "config.h" #include "logger.h" -#include "services.h" +#include "services.h" #include "walle.h" #include <library/cpp/actors/core/hfunc.h> @@ -31,7 +31,7 @@ public: struct TEvPrivate { enum EEv { EvClusterInfo = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvUpdateClusterInfo, + EvUpdateClusterInfo, EvCleanupExpired, EvCleanupWalle, EvLogAndSend, @@ -52,8 +52,8 @@ public: } }; - struct TEvUpdateClusterInfo : public TEventLocal<TEvUpdateClusterInfo, EvUpdateClusterInfo> {}; - + struct TEvUpdateClusterInfo : public TEventLocal<TEvUpdateClusterInfo, EvUpdateClusterInfo> {}; + struct TEvCleanupExpired : public TEventLocal<TEvCleanupExpired, EvCleanupExpired> {}; struct TEvCleanupWalle : public TEventLocal<TEvCleanupWalle, EvCleanupWalle> {}; @@ -151,44 +151,44 @@ private: ITransaction *CreateTxProcessNotification(TEvCms::TEvNotification::TPtr &ev); ITransaction *CreateTxRejectNotification(TEvCms::TEvManageNotificationRequest::TPtr &ev); ITransaction *CreateTxRemoveExpiredNotifications(); - ITransaction *CreateTxRemoveRequest(const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp); - ITransaction *CreateTxRemovePermissions(TVector<TString> ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired = false); - ITransaction *CreateTxRemoveWalleTask(const TString &id); - ITransaction *CreateTxStorePermissions(THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, - const TString &owner, TAutoPtr<TRequestInfo> scheduled); - ITransaction *CreateTxStoreWalleTask(const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp); + ITransaction *CreateTxRemoveRequest(const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp); + ITransaction *CreateTxRemovePermissions(TVector<TString> ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired = false); + ITransaction *CreateTxRemoveWalleTask(const TString &id); + ITransaction *CreateTxStorePermissions(THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, + const TString &owner, TAutoPtr<TRequestInfo> scheduled); + ITransaction *CreateTxStoreWalleTask(const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp); ITransaction *CreateTxUpdateConfig(TEvCms::TEvSetConfigRequest::TPtr &ev); ITransaction *CreateTxUpdateConfig(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); ITransaction *CreateTxUpdateDowntimes(); - static void AuditLog(const TActorContext& ctx, const TString& message) { - NCms::AuditLog("CMS tablet", message, ctx); - } + static void AuditLog(const TActorContext& ctx, const TString& message) { + NCms::AuditLog("CMS tablet", message, ctx); + } - static void AuditLog(const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { - return AuditLog(ctx, TStringBuilder() << "Reply" - << ": request# " << request->ToString() - << ", response# " << response->ToString()); - } + static void AuditLog(const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { + return AuditLog(ctx, TStringBuilder() << "Reply" + << ": request# " << request->ToString() + << ", response# " << response->ToString()); + } - static void Reply(const IEventBase* request, TAutoPtr<IEventHandle> response, const TActorContext& ctx) { - AuditLog(request, response->GetBase(), ctx); - ctx.Send(response); + static void Reply(const IEventBase* request, TAutoPtr<IEventHandle> response, const TActorContext& ctx) { + AuditLog(request, response->GetBase(), ctx); + ctx.Send(response); } - template <typename TEvRequestPtr> - static void Reply(TEvRequestPtr& request, THolder<IEventBase> response, const TActorContext& ctx) { - AuditLog(request->Get(), response.Get(), ctx); - ctx.Send(request->Sender, response.Release(), 0, request->Cookie); - } + template <typename TEvRequestPtr> + static void Reply(TEvRequestPtr& request, THolder<IEventBase> response, const TActorContext& ctx) { + AuditLog(request->Get(), response.Get(), ctx); + ctx.Send(request->Sender, response.Release(), 0, request->Cookie); + } - template <typename TEvResponse, typename TEvRequestPtr> - static void ReplyWithError(TEvRequestPtr& ev, EStatusCode code, const TString& reason, const TActorContext& ctx) { - auto response = MakeHolder<TEvResponse>(); - response->Record.MutableStatus()->SetCode(code); - response->Record.MutableStatus()->SetReason(reason); + template <typename TEvResponse, typename TEvRequestPtr> + static void ReplyWithError(TEvRequestPtr& ev, EStatusCode code, const TString& reason, const TActorContext& ctx) { + auto response = MakeHolder<TEvResponse>(); + response->Record.MutableStatus()->SetCode(code); + response->Record.MutableStatus()->SetReason(reason); - Reply<TEvRequestPtr>(ev, std::move(response), ctx); + Reply<TEvRequestPtr>(ev, std::move(response), ctx); } STFUNC(StateInit) @@ -198,9 +198,9 @@ private: StateInitImpl(ev, ctx); } - template <typename TEvRequest, typename TEvResponse> - void HandleNotSupported(typename TEvRequest::TPtr& ev, const TActorContext& ctx) { - ReplyWithError<TEvResponse, typename TEvRequest::TPtr>(ev, NKikimrCms::TStatus::ERROR, NotSupportedReason, ctx); + template <typename TEvRequest, typename TEvResponse> + void HandleNotSupported(typename TEvRequest::TPtr& ev, const TActorContext& ctx) { + ReplyWithError<TEvResponse, typename TEvRequest::TPtr>(ev, NKikimrCms::TStatus::ERROR, NotSupportedReason, ctx); } STFUNC(StateNotSupported) @@ -238,7 +238,7 @@ private: switch (ev->GetTypeRewrite()) { HFunc(TEvPrivate::TEvClusterInfo, Handle); HFunc(TEvPrivate::TEvLogAndSend, Handle); - FFunc(TEvPrivate::EvUpdateClusterInfo, EnqueueRequest); + FFunc(TEvPrivate::EvUpdateClusterInfo, EnqueueRequest); CFunc(TEvPrivate::EvCleanupExpired, CleanupExpired); CFunc(TEvPrivate::EvCleanupLog, CleanupLog); CFunc(TEvPrivate::EvCleanupWalle, CleanupWalleTasks); @@ -258,8 +258,8 @@ private: HFunc(TEvCms::TEvRemoveWalleTask, Handle); HFunc(TEvCms::TEvGetConfigRequest, Handle); HFunc(TEvCms::TEvSetConfigRequest, Handle); - HFunc(TEvCms::TEvResetMarkerRequest, Handle); - HFunc(TEvCms::TEvSetMarkerRequest, Handle); + HFunc(TEvCms::TEvResetMarkerRequest, Handle); + HFunc(TEvCms::TEvSetMarkerRequest, Handle); HFunc(TEvCms::TEvGetLogTailRequest, Handle); HFunc(TEvConsole::TEvConfigNotificationRequest, Handle); HFunc(TEvConsole::TEvReplaceConfigSubscriptionsResponse, Handle); @@ -299,40 +299,40 @@ private: NKikimrCms::TStatus::ECode &code, TString &error, const TActorContext &ctx); - bool CheckAction(const NKikimrCms::TAction &action, const TActionOptions &options, + bool CheckAction(const NKikimrCms::TAction &action, const TActionOptions &options, TErrorInfo &error) const; - bool CheckActionShutdownNode(const NKikimrCms::TAction &action, - const TActionOptions &options, - const TNodeInfo &node, - TErrorInfo &error) const; + bool CheckActionShutdownNode(const NKikimrCms::TAction &action, + const TActionOptions &options, + const TNodeInfo &node, + TErrorInfo &error) const; bool CheckActionRestartServices(const NKikimrCms::TAction &action, - const TActionOptions &options, + const TActionOptions &options, TErrorInfo &error) const; bool CheckActionShutdownHost(const NKikimrCms::TAction &action, - const TActionOptions &options, + const TActionOptions &options, TErrorInfo &error) const; bool CheckActionReplaceDevices(const NKikimrCms::TAction &action, - const TActionOptions &options, + const TActionOptions &options, TErrorInfo &error) const; - bool TryToLockNode(const NKikimrCms::TAction &action, - const TActionOptions &options, - const TNodeInfo &node, + bool TryToLockNode(const NKikimrCms::TAction &action, + const TActionOptions &options, + const TNodeInfo &node, TErrorInfo& error) const; bool TryToLockPDisk(const NKikimrCms::TAction &action, - const TActionOptions &options, + const TActionOptions &options, const TPDiskInfo &pdisk, TErrorInfo &error) const; bool TryToLockVDisks(const NKikimrCms::TAction &action, - const TActionOptions &options, + const TActionOptions &options, const TSet<TVDiskID> &vdisks, TErrorInfo &error) const; - bool TryToLockVDisk(const TActionOptions &options, + bool TryToLockVDisk(const TActionOptions &options, const TVDiskInfo &vdisk, TDuration duration, TErrorInfo &error) const; void AcceptPermissions(NKikimrCms::TPermissionResponse &resp, const TString &requestId, - const TString &owner, const TActorContext &ctx, bool check = false); - void ScheduleUpdateClusterInfo(const TActorContext &ctx, bool now = false); + const TString &owner, const TActorContext &ctx, bool check = false); + void ScheduleUpdateClusterInfo(const TActorContext &ctx, bool now = false); void ScheduleCleanup(TInstant time, const TActorContext &ctx); void SchedulePermissionsCleanup(const TActorContext &ctx); void ScheduleNotificationsCleanup(const TActorContext &ctx); @@ -385,7 +385,7 @@ private: void Handle(TEvPrivate::TEvClusterInfo::TPtr &ev, const TActorContext &ctx); void Handle(TEvPrivate::TEvLogAndSend::TPtr &ev, const TActorContext &ctx); - void Handle(TEvPrivate::TEvUpdateClusterInfo::TPtr &ev, const TActorContext &ctx); + void Handle(TEvPrivate::TEvUpdateClusterInfo::TPtr &ev, const TActorContext &ctx); void Handle(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvCms::TEvManagePermissionRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvCms::TEvClusterStateRequest::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/cms/cms_state.h b/ydb/core/cms/cms_state.h index a264fed4b30..5f0b6ff2e21 100644 --- a/ydb/core/cms/cms_state.h +++ b/ydb/core/cms/cms_state.h @@ -5,8 +5,8 @@ #include <util/generic/set.h> #include <util/generic/string.h> -#include <util/string/builder.h> -#include <util/string/join.h> +#include <util/string/builder.h> +#include <util/string/join.h> namespace NKikimr { namespace NCms { @@ -15,14 +15,14 @@ struct TWalleTaskInfo { TString TaskId; TString RequestId; TSet<TString> Permissions; - - TString ToString() const { - return TStringBuilder() << "{" - << " TaskId: " << TaskId - << " RequestId: " << RequestId - << " Permissions: [" << JoinSeq(", ", Permissions) << "]" - << " }"; - } + + TString ToString() const { + return TStringBuilder() << "{" + << " TaskId: " << TaskId + << " RequestId: " << RequestId + << " Permissions: [" << JoinSeq(", ", Permissions) << "]" + << " }"; + } }; struct TCmsState : public TAtomicRefCount<TCmsState> { diff --git a/ydb/core/cms/cms_tenants_ut.cpp b/ydb/core/cms/cms_tenants_ut.cpp index e70a8b60f98..e14fe4dd574 100644 --- a/ydb/core/cms/cms_tenants_ut.cpp +++ b/ydb/core/cms/cms_tenants_ut.cpp @@ -349,18 +349,18 @@ Y_UNIT_TEST_SUITE(TCmsTenatsTest) { TestScheduledPermission(true); } - void TestShutdownHost(bool usePolicy) + void TestShutdownHost(bool usePolicy) { TNodeTenantsMap staticTenants; staticTenants[0].push_back("user0"); staticTenants[8].push_back("user0"); - TCmsTestEnv env(16, 1, staticTenants); + TCmsTestEnv env(16, 1, staticTenants); env.SetLimits(0, 10, 0, 0); - env.CheckPermissionRequest("user", false, false, false, usePolicy, - TStatus::ALLOW, + env.CheckPermissionRequest("user", false, false, false, usePolicy, + TStatus::ALLOW, MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(0), 60000000)); env.CheckPermissionRequest("user", false, false, false, usePolicy, @@ -368,63 +368,63 @@ Y_UNIT_TEST_SUITE(TCmsTenatsTest) { MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(8), 60000000)); env.CheckPermissionRequest("user", false, false, false, usePolicy, - TStatus::ALLOW, + TStatus::ALLOW, MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(15), 60000000)); } - Y_UNIT_TEST(RequestShutdownHost) { - TestShutdownHost(false); + Y_UNIT_TEST(RequestShutdownHost) { + TestShutdownHost(false); + } + + Y_UNIT_TEST(RequestShutdownHostWithTenantPolicy) { + TestShutdownHost(true); } - Y_UNIT_TEST(RequestShutdownHostWithTenantPolicy) { - TestShutdownHost(true); + Y_UNIT_TEST(RequestRestartServices) { + TCmsTestEnv env(16, 1, TNodeTenantsMap{ + {8, {"user0"}}, + {9, {"user0"}}, + {10, {"user0"}}, + {11, {"user0"}}, + {12, {"user0"}}, + {13, {"user0"}}, + {14, {"user0"}}, + {15, {"user0"}}, + }); + + // there is not storage on dynamic nodes + env.CheckPermissionRequest("user", false, false, false, true, TStatus::NO_SUCH_SERVICE, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(8), 60000000, "storage")); + + env.SetLimits(0, 50, 0, 0); + // it is allowed to restart 50% + env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(8), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(9), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(10), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(11), 60000000, "dynnode") + ); + env.CheckPermissionRequest("user", false, false, false, true, TStatus::DISALLOW_TEMP, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(12), 60000000, "dynnode")); + + env.SetLimits(0, 0, 0, 0); + // it is allowed to restart them all + env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(12), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(13), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(14), 60000000, "dynnode"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(15), 60000000, "dynnode") + ); + + // it doesn't affect storage nodes + env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 60000000, "storage")); + + // and fault model checks still work + env.CheckPermissionRequest("user", false, false, false, true, TStatus::DISALLOW_TEMP, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(1), 60000000, "storage")); } - Y_UNIT_TEST(RequestRestartServices) { - TCmsTestEnv env(16, 1, TNodeTenantsMap{ - {8, {"user0"}}, - {9, {"user0"}}, - {10, {"user0"}}, - {11, {"user0"}}, - {12, {"user0"}}, - {13, {"user0"}}, - {14, {"user0"}}, - {15, {"user0"}}, - }); - - // there is not storage on dynamic nodes - env.CheckPermissionRequest("user", false, false, false, true, TStatus::NO_SUCH_SERVICE, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(8), 60000000, "storage")); - - env.SetLimits(0, 50, 0, 0); - // it is allowed to restart 50% - env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(8), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(9), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(10), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(11), 60000000, "dynnode") - ); - env.CheckPermissionRequest("user", false, false, false, true, TStatus::DISALLOW_TEMP, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(12), 60000000, "dynnode")); - - env.SetLimits(0, 0, 0, 0); - // it is allowed to restart them all - env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(12), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(13), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(14), 60000000, "dynnode"), - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(15), 60000000, "dynnode") - ); - - // it doesn't affect storage nodes - env.CheckPermissionRequest("user", false, false, false, true, TStatus::ALLOW, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 60000000, "storage")); - - // and fault model checks still work - env.CheckPermissionRequest("user", false, false, false, true, TStatus::DISALLOW_TEMP, - MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(1), 60000000, "storage")); - } - void TestLimitForceRestartMode(bool tenant, bool ratio) { TNodeTenantsMap staticTenants; diff --git a/ydb/core/cms/cms_tx_load_state.cpp b/ydb/core/cms/cms_tx_load_state.cpp index 4fe0d380629..eb144dc0def 100644 --- a/ydb/core/cms/cms_tx_load_state.cpp +++ b/ydb/core/cms/cms_tx_load_state.cpp @@ -37,7 +37,7 @@ public: if (!paramRow.IsReady() || !permissionRowset.IsReady() || !requestRowset.IsReady() || !walleTaskRowset.IsReady() - || !notificationRowset.IsReady() || !logRowset.IsReady()) + || !notificationRowset.IsReady() || !logRowset.IsReady()) return false; NKikimrCms::TCmsConfig config; @@ -193,7 +193,7 @@ public: Self->SchedulePermissionsCleanup(ctx); Self->ScheduleNotificationsCleanup(ctx); Self->ScheduleLogCleanup(ctx); - Self->ScheduleUpdateClusterInfo(ctx, true); + Self->ScheduleUpdateClusterInfo(ctx, true); Self->ProcessInitQueue(ctx); } }; diff --git a/ydb/core/cms/cms_tx_process_notification.cpp b/ydb/core/cms/cms_tx_process_notification.cpp index ca8565c408e..2174a82f171 100644 --- a/ydb/core/cms/cms_tx_process_notification.cpp +++ b/ydb/core/cms/cms_tx_process_notification.cpp @@ -21,7 +21,7 @@ public: auto &rec = Event->Get()->Record; Response = new TEvCms::TEvNotificationResponse; - LOG_INFO(ctx, NKikimrServices::CMS, "Processing notification from %s (time=%s reason='%s')", + LOG_INFO(ctx, NKikimrServices::CMS, "Processing notification from %s (time=%s reason='%s')", rec.GetUser().data(), TInstant::MicroSeconds(rec.GetTime()).ToStringLocalUpToSeconds().data(), rec.GetReason().data()); @@ -39,10 +39,10 @@ public: auto row = db.Table<Schema::Notification>().Key(id); row.Update(NIceDb::TUpdate<Schema::Notification::Owner>(rec.GetUser()), NIceDb::TUpdate<Schema::Notification::NotificationProto>(notificationStr)); - - Self->AuditLog(ctx, TStringBuilder() << "Store notification" - << ": id# " << id - << ", body# " << notificationStr); + + Self->AuditLog(ctx, TStringBuilder() << "Store notification" + << ": id# " << id + << ", body# " << notificationStr); } return true; diff --git a/ydb/core/cms/cms_tx_reject_notification.cpp b/ydb/core/cms/cms_tx_reject_notification.cpp index 3b9f17a9fb5..5612dd9fbb1 100644 --- a/ydb/core/cms/cms_tx_reject_notification.cpp +++ b/ydb/core/cms/cms_tx_reject_notification.cpp @@ -28,9 +28,9 @@ public: TErrorInfo error; if (Self->RemoveNotification(id, user, !dry, error)) { if (!dry) { - Self->AuditLog(ctx, TStringBuilder() << "Remove notification" - << ": id# " << id - << ", reason# " << "explicit remove"); + Self->AuditLog(ctx, TStringBuilder() << "Remove notification" + << ": id# " << id + << ", reason# " << "explicit remove"); NIceDb::TNiceDb db(txc.DB); db.Table<Schema::Notification>().Key(id).Delete(); @@ -42,7 +42,7 @@ public: Response->Record.MutableStatus()->SetReason(error.Reason); } - LOG_INFO(ctx, NKikimrServices::CMS, "Response status: %s %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Response status: %s %s", ToString(Response->Record.GetStatus().GetCode()).data(), Response->Record.GetStatus().GetReason().data()); diff --git a/ydb/core/cms/cms_tx_remove_expired_notifications.cpp b/ydb/core/cms/cms_tx_remove_expired_notifications.cpp index bb79e68e261..6235b7f8f0f 100644 --- a/ydb/core/cms/cms_tx_remove_expired_notifications.cpp +++ b/ydb/core/cms/cms_tx_remove_expired_notifications.cpp @@ -31,7 +31,7 @@ public: TInstant deadline = time + TDuration::MicroSeconds(i->GetDuration()); if (deadline <= now) { - LOG_INFO(ctx, NKikimrServices::CMS, "Removing expired action from notification %s: %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Removing expired action from notification %s: %s", info.NotificationId.data(), i->ShortDebugString().data()); i = actions->erase(i); @@ -41,9 +41,9 @@ public: } if (actions->empty()) { - Self->AuditLog(ctx, TStringBuilder() << "Remove notification" - << ": id# " << info.NotificationId - << ", reason# " << "scheduled cleanup"); + Self->AuditLog(ctx, TStringBuilder() << "Remove notification" + << ": id# " << info.NotificationId + << ", reason# " << "scheduled cleanup"); NIceDb::TNiceDb db(txc.DB); db.Table<Schema::Notification>().Key(info.NotificationId).Delete(); @@ -57,9 +57,9 @@ public: auto row = db.Table<Schema::Notification>().Key(info.NotificationId); row.Update(NIceDb::TUpdate<Schema::Notification::NotificationProto>(notificationStr)); - Self->AuditLog(ctx, TStringBuilder() << "Update notification" - << ": id# " << info.NotificationId - << ", body# " << notificationStr); + Self->AuditLog(ctx, TStringBuilder() << "Update notification" + << ": id# " << info.NotificationId + << ", body# " << notificationStr); } entry = next; diff --git a/ydb/core/cms/cms_tx_remove_permissions.cpp b/ydb/core/cms/cms_tx_remove_permissions.cpp index f16b2ee3a3c..6e638c72ffb 100644 --- a/ydb/core/cms/cms_tx_remove_permissions.cpp +++ b/ydb/core/cms/cms_tx_remove_permissions.cpp @@ -8,12 +8,12 @@ namespace NCms { class TCms::TTxRemovePermissions : public TTransactionBase<TCms> { public: - TTxRemovePermissions(TCms *self, TVector<TString> &&ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired) + TTxRemovePermissions(TCms *self, TVector<TString> &&ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired) : TBase(self) - , Request(std::move(req)) + , Request(std::move(req)) , Response(resp) , Ids(ids) - , Expired(expired) + , Expired(expired) { } @@ -30,23 +30,23 @@ public: Self->State->Permissions.erase(id); db.Table<Schema::Permission>().Key(id).Delete(); - if (Expired && Self->State->ScheduledRequests.contains(requestId)) { - Self->State->ScheduledRequests.erase(requestId); - db.Table<Schema::Request>().Key(requestId).Delete(); - - Self->AuditLog(ctx, TStringBuilder() << "Remove request" - << ": id# " << requestId - << ", reason# " << "permission " << id << " has expired"); - } - + if (Expired && Self->State->ScheduledRequests.contains(requestId)) { + Self->State->ScheduledRequests.erase(requestId); + db.Table<Schema::Request>().Key(requestId).Delete(); + + Self->AuditLog(ctx, TStringBuilder() << "Remove request" + << ": id# " << requestId + << ", reason# " << "permission " << id << " has expired"); + } + if (Self->State->WalleRequests.contains(requestId)) { auto taskId = Self->State->WalleRequests.find(requestId)->second; Self->State->WalleTasks.find(taskId)->second.Permissions.erase(id); } - Self->AuditLog(ctx, TStringBuilder() << "Remove permission" - << ": id# " << id - << ", reason# " << (Request ? "explicit remove" : "scheduled cleanup")); + Self->AuditLog(ctx, TStringBuilder() << "Remove permission" + << ": id# " << id + << ", reason# " << (Request ? "explicit remove" : "scheduled cleanup")); } return true; @@ -57,23 +57,23 @@ public: LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxRemovePermissions Complete"); if (Response) { - Y_VERIFY(Request); - Self->Reply(Request.Get(), Response, ctx); + Y_VERIFY(Request); + Self->Reply(Request.Get(), Response, ctx); } Self->RemoveEmptyWalleTasks(ctx); } private: - THolder<IEventBase> Request; + THolder<IEventBase> Request; TAutoPtr<IEventHandle> Response; TVector<TString> Ids; - bool Expired; + bool Expired; }; -ITransaction* TCms::CreateTxRemovePermissions(TVector<TString> ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired) +ITransaction* TCms::CreateTxRemovePermissions(TVector<TString> ids, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, bool expired) { - return new TTxRemovePermissions(this, std::move(ids), std::move(req), std::move(resp), expired); + return new TTxRemovePermissions(this, std::move(ids), std::move(req), std::move(resp), expired); } } // NCms diff --git a/ydb/core/cms/cms_tx_remove_request.cpp b/ydb/core/cms/cms_tx_remove_request.cpp index 92fa27c4f89..fab8144cfe3 100644 --- a/ydb/core/cms/cms_tx_remove_request.cpp +++ b/ydb/core/cms/cms_tx_remove_request.cpp @@ -8,9 +8,9 @@ namespace NCms { class TCms::TTxRemoveRequest : public TTransactionBase<TCms> { public: - TTxRemoveRequest(TCms *self, const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) + TTxRemoveRequest(TCms *self, const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) : TBase(self) - , Request(std::move(req)) + , Request(std::move(req)) , Response(resp) , Id(id) { @@ -24,9 +24,9 @@ public: db.Table<Schema::Request>().Key(Id).Delete(); Self->State->ScheduledRequests.erase(Id); - Self->AuditLog(ctx, TStringBuilder() << "Remove request" - << ": id# " << Id - << ", reason# " << (Request ? "explicit remove" : "scheduled cleanup")); + Self->AuditLog(ctx, TStringBuilder() << "Remove request" + << ": id# " << Id + << ", reason# " << (Request ? "explicit remove" : "scheduled cleanup")); return true; } @@ -36,22 +36,22 @@ public: LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxRemoveRequest Complete"); if (Response) { - Y_VERIFY(Request); - Self->Reply(Request.Get(), Response, ctx); + Y_VERIFY(Request); + Self->Reply(Request.Get(), Response, ctx); } Self->RemoveEmptyWalleTasks(ctx); } private: - THolder<IEventBase> Request; + THolder<IEventBase> Request; TAutoPtr<IEventHandle> Response; TString Id; }; -ITransaction* TCms::CreateTxRemoveRequest(const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) +ITransaction* TCms::CreateTxRemoveRequest(const TString &id, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) { - return new TTxRemoveRequest(this, id, std::move(req), std::move(resp)); + return new TTxRemoveRequest(this, id, std::move(req), std::move(resp)); } } // NCms diff --git a/ydb/core/cms/cms_tx_remove_walle_task.cpp b/ydb/core/cms/cms_tx_remove_walle_task.cpp index 0df9000a5c7..d9e099f0db7 100644 --- a/ydb/core/cms/cms_tx_remove_walle_task.cpp +++ b/ydb/core/cms/cms_tx_remove_walle_task.cpp @@ -8,7 +8,7 @@ namespace NCms { class TCms::TTxRemoveWalleTask : public TTransactionBase<TCms> { public: - TTxRemoveWalleTask(TCms *self, const TString &id) + TTxRemoveWalleTask(TCms *self, const TString &id) : TBase(self) , Id(id) { @@ -26,11 +26,11 @@ public: Self->State->WalleRequests.erase(it->second.RequestId); Self->State->WalleTasks.erase(it); - Self->AuditLog(ctx, TStringBuilder() << "Remove wall-e task" - << ": id# " << Id); - } else { + Self->AuditLog(ctx, TStringBuilder() << "Remove wall-e task" + << ": id# " << Id); + } else { LOG_ERROR(ctx, NKikimrServices::CMS, "Can't find Wall-E task %s", Id.data()); - } + } return true; } @@ -44,9 +44,9 @@ private: TString Id; }; -ITransaction* TCms::CreateTxRemoveWalleTask(const TString &id) +ITransaction* TCms::CreateTxRemoveWalleTask(const TString &id) { - return new TTxRemoveWalleTask(this, id); + return new TTxRemoveWalleTask(this, id); } } // NCms diff --git a/ydb/core/cms/cms_tx_store_permissions.cpp b/ydb/core/cms/cms_tx_store_permissions.cpp index ecde26aec3d..fc5bc54e3e6 100644 --- a/ydb/core/cms/cms_tx_store_permissions.cpp +++ b/ydb/core/cms/cms_tx_store_permissions.cpp @@ -8,11 +8,11 @@ namespace NCms { class TCms::TTxStorePermissions : public TTransactionBase<TCms> { public: - TTxStorePermissions(TCms *self, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, - const TString &owner, TAutoPtr<TRequestInfo> scheduled) + TTxStorePermissions(TCms *self, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, + const TString &owner, TAutoPtr<TRequestInfo> scheduled) : TBase(self) - , Request(std::move(req)) - , Response(std::move(resp)) + , Request(std::move(req)) + , Response(std::move(resp)) , Owner(owner) , Scheduled(scheduled) , NextPermissionId(self->State->NextPermissionId) @@ -28,8 +28,8 @@ public: db.Table<Schema::Param>().Key(1).Update(NIceDb::TUpdate<Schema::Param::NextPermissionID>(NextPermissionId), NIceDb::TUpdate<Schema::Param::NextRequestID>(NextRequestId)); - const auto& rec = Response->Get<TEvCms::TEvPermissionResponse>()->Record; - for (const auto &permission : rec.GetPermissions()) { + const auto& rec = Response->Get<TEvCms::TEvPermissionResponse>()->Record; + for (const auto &permission : rec.GetPermissions()) { const auto &id = permission.GetId(); const auto &requestId = Scheduled ? Scheduled->RequestId : ""; ui64 deadline = permission.GetDeadline(); @@ -42,10 +42,10 @@ public: NIceDb::TUpdate<Schema::Permission::Deadline>(deadline), NIceDb::TUpdate<Schema::Permission::RequestID>(requestId)); - Self->AuditLog(ctx, TStringBuilder() << "Store permission" - << ": id# " << id - << ", validity# " << TInstant::MicroSeconds(deadline) - << ", action# " << actionStr); + Self->AuditLog(ctx, TStringBuilder() << "Store permission" + << ": id# " << id + << ", validity# " << TInstant::MicroSeconds(deadline) + << ", action# " << actionStr); } if (Scheduled) { @@ -62,17 +62,17 @@ public: NIceDb::TUpdate<Schema::Request::Order>(order), NIceDb::TUpdate<Schema::Request::Content>(requestStr)); - Self->AuditLog(ctx, TStringBuilder() << "Store request" - << ": id# " << id - << ", owner# " << owner - << ", order# " << order - << ", body# " << requestStr); + Self->AuditLog(ctx, TStringBuilder() << "Store request" + << ": id# " << id + << ", owner# " << owner + << ", order# " << order + << ", body# " << requestStr); } else { db.Table<Schema::Request>().Key(id).Delete(); - Self->AuditLog(ctx, TStringBuilder() << "Remove request" - << ": id# " << id - << ", owner# " << owner); + Self->AuditLog(ctx, TStringBuilder() << "Remove request" + << ": id# " << id + << ", owner# " << owner); } } @@ -84,23 +84,23 @@ public: { LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxStorePermissions complete"); - Self->Reply(Request.Get(), Response, ctx); + Self->Reply(Request.Get(), Response, ctx); Self->SchedulePermissionsCleanup(ctx); } private: - THolder<IEventBase> Request; - TAutoPtr<IEventHandle> Response; + THolder<IEventBase> Request; + TAutoPtr<IEventHandle> Response; TString Owner; TAutoPtr<TRequestInfo> Scheduled; ui64 NextPermissionId; ui64 NextRequestId; }; -ITransaction* TCms::CreateTxStorePermissions(THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, - const TString &owner, TAutoPtr<TRequestInfo> scheduled) +ITransaction* TCms::CreateTxStorePermissions(THolder<IEventBase> req, TAutoPtr<IEventHandle> resp, + const TString &owner, TAutoPtr<TRequestInfo> scheduled) { - return new TTxStorePermissions(this, std::move(req), std::move(resp), owner, std::move(scheduled)); + return new TTxStorePermissions(this, std::move(req), std::move(resp), owner, std::move(scheduled)); } } // NCms diff --git a/ydb/core/cms/cms_tx_store_walle_task.cpp b/ydb/core/cms/cms_tx_store_walle_task.cpp index 273dae9ac8b..f05828477a4 100644 --- a/ydb/core/cms/cms_tx_store_walle_task.cpp +++ b/ydb/core/cms/cms_tx_store_walle_task.cpp @@ -8,15 +8,15 @@ namespace NCms { class TCms::TTxStoreWalleTask : public TTransactionBase<TCms> { public: - TTxStoreWalleTask(TCms *self, const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) + TTxStoreWalleTask(TCms *self, const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) : TBase(self) , TaskId(task.TaskId) , RequestId(task.RequestId) - , Request(std::move(req)) + , Request(std::move(req)) , Response(std::move(resp)) { - Y_VERIFY(Request); - Y_VERIFY(Response); + Y_VERIFY(Request); + Y_VERIFY(Response); } bool Execute(TTransactionContext &txc, const TActorContext &ctx) override @@ -27,29 +27,29 @@ public: auto row = db.Table<Schema::WalleTask>().Key(TaskId); row.Update(NIceDb::TUpdate<Schema::WalleTask::RequestID>(RequestId)); - Self->AuditLog(ctx, TStringBuilder() << "Store wall-e task" - << ": id# " << TaskId - << ", requestId# " << RequestId); - + Self->AuditLog(ctx, TStringBuilder() << "Store wall-e task" + << ": id# " << TaskId + << ", requestId# " << RequestId); + return true; } void Complete(const TActorContext &ctx) override { LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxStoreWalleTask Complete"); - Self->Reply(Request.Get(), Response, ctx); + Self->Reply(Request.Get(), Response, ctx); } private: TString TaskId; TString RequestId; - THolder<IEventBase> Request; + THolder<IEventBase> Request; TAutoPtr<IEventHandle> Response; }; -ITransaction *TCms::CreateTxStoreWalleTask(const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) +ITransaction *TCms::CreateTxStoreWalleTask(const TWalleTaskInfo &task, THolder<IEventBase> req, TAutoPtr<IEventHandle> resp) { - return new TTxStoreWalleTask(this, task, std::move(req), std::move(resp)); + return new TTxStoreWalleTask(this, task, std::move(req), std::move(resp)); } } // NCms diff --git a/ydb/core/cms/cms_tx_update_config.cpp b/ydb/core/cms/cms_tx_update_config.cpp index c3c8b602f20..59f962b3e2c 100644 --- a/ydb/core/cms/cms_tx_update_config.cpp +++ b/ydb/core/cms/cms_tx_update_config.cpp @@ -1,6 +1,6 @@ #include "cms_impl.h" #include "scheme.h" -#include "sentinel.h" +#include "sentinel.h" namespace NKikimr { namespace NCms { @@ -53,16 +53,16 @@ public: ctx.Send(Response.Release()); } - if (Self->State->Config.SentinelConfig.Enable) { - if (!Self->State->Sentinel) { + if (Self->State->Config.SentinelConfig.Enable) { + if (!Self->State->Sentinel) { Self->State->Sentinel = Self->RegisterWithSameMailbox(CreateSentinel(Self->State)); - } - } else { - if (Self->State->Sentinel) { - ctx.Send(Self->State->Sentinel, new TEvents::TEvPoisonPill()); + } + } else { + if (Self->State->Sentinel) { + ctx.Send(Self->State->Sentinel, new TEvents::TEvPoisonPill()); Self->State->Sentinel = TActorId(); - } - } + } + } } private: diff --git a/ydb/core/cms/cms_ut.cpp b/ydb/core/cms/cms_ut.cpp index 3904e1c4472..9842fb71567 100644 --- a/ydb/core/cms/cms_ut.cpp +++ b/ydb/core/cms/cms_ut.cpp @@ -145,28 +145,28 @@ Y_UNIT_TEST_SUITE(TCmsTest) { Y_UNIT_TEST(StateRequestNode) { - TCmsTestEnv env(8, TNodeTenantsMap{{1, {"user0"}}}); - - THashMap<ui32, TString> nodeIdxToServiceName = { - {0, "storage"}, - {1, "dynnode"}, - }; - - for (const auto& [nodeIdx, serviceName] : nodeIdxToServiceName) { - NKikimrCms::TClusterStateRequest request; - request.AddHosts(ToString(env.GetNodeId(nodeIdx))); - - auto state = env.RequestState(request); - UNIT_ASSERT_VALUES_EQUAL(state.HostsSize(), 1); - const auto &host = state.GetHosts(0); - UNIT_ASSERT_VALUES_EQUAL(host.GetName(), "::1"); - UNIT_ASSERT_VALUES_EQUAL(host.GetState(), UP); - UNIT_ASSERT_VALUES_EQUAL(host.ServicesSize(), 1); - const auto &service = host.GetServices(0); - UNIT_ASSERT_VALUES_EQUAL(service.GetName(), serviceName); - UNIT_ASSERT_VALUES_EQUAL(service.GetState(), UP); - UNIT_ASSERT_VALUES_EQUAL(service.GetVersion(), ToString(GetProgramSvnRevision())); - } + TCmsTestEnv env(8, TNodeTenantsMap{{1, {"user0"}}}); + + THashMap<ui32, TString> nodeIdxToServiceName = { + {0, "storage"}, + {1, "dynnode"}, + }; + + for (const auto& [nodeIdx, serviceName] : nodeIdxToServiceName) { + NKikimrCms::TClusterStateRequest request; + request.AddHosts(ToString(env.GetNodeId(nodeIdx))); + + auto state = env.RequestState(request); + UNIT_ASSERT_VALUES_EQUAL(state.HostsSize(), 1); + const auto &host = state.GetHosts(0); + UNIT_ASSERT_VALUES_EQUAL(host.GetName(), "::1"); + UNIT_ASSERT_VALUES_EQUAL(host.GetState(), UP); + UNIT_ASSERT_VALUES_EQUAL(host.ServicesSize(), 1); + const auto &service = host.GetServices(0); + UNIT_ASSERT_VALUES_EQUAL(service.GetName(), serviceName); + UNIT_ASSERT_VALUES_EQUAL(service.GetState(), UP); + UNIT_ASSERT_VALUES_EQUAL(service.GetVersion(), ToString(GetProgramSvnRevision())); + } } Y_UNIT_TEST(StateRequestUnknownNode) @@ -695,29 +695,29 @@ Y_UNIT_TEST_SUITE(TCmsTest) { env.CheckWalleListTasks(0); } - Y_UNIT_TEST(WalleTasksWithNodeLimit) - { - TCmsTestEnv env(24, 4); - - // set limit - NKikimrCms::TCmsConfig config; - config.MutableClusterLimits()->SetDisabledNodesLimit(1); - env.SetCmsConfig(config); - - // ok - env.CheckWalleCreateTask("task-1", "reboot", false, TStatus::ALLOW, env.GetNodeId(0)); - // schedule - env.CheckWalleCreateTask("task-2", "reboot", false, TStatus::DISALLOW_TEMP, env.GetNodeId(1)); - // remove first task - env.CheckWalleRemoveTask("task-1"); - // ok for second task - env.CheckWalleCheckTask("task-2", TStatus::ALLOW, env.GetNodeId(1)); - - // check task is not removed - env.AdvanceCurrentTime(TDuration::Minutes(20)); - env.CheckWalleCheckTask("task-2", TStatus::ALLOW, env.GetNodeId(1)); - } - + Y_UNIT_TEST(WalleTasksWithNodeLimit) + { + TCmsTestEnv env(24, 4); + + // set limit + NKikimrCms::TCmsConfig config; + config.MutableClusterLimits()->SetDisabledNodesLimit(1); + env.SetCmsConfig(config); + + // ok + env.CheckWalleCreateTask("task-1", "reboot", false, TStatus::ALLOW, env.GetNodeId(0)); + // schedule + env.CheckWalleCreateTask("task-2", "reboot", false, TStatus::DISALLOW_TEMP, env.GetNodeId(1)); + // remove first task + env.CheckWalleRemoveTask("task-1"); + // ok for second task + env.CheckWalleCheckTask("task-2", TStatus::ALLOW, env.GetNodeId(1)); + + // check task is not removed + env.AdvanceCurrentTime(TDuration::Minutes(20)); + env.CheckWalleCheckTask("task-2", TStatus::ALLOW, env.GetNodeId(1)); + } + Y_UNIT_TEST(Notifications) { TCmsTestEnv env(8); @@ -795,9 +795,9 @@ Y_UNIT_TEST_SUITE(TCmsTest) { MakeAction(TAction::REPLACE_DEVICES, env.GetNodeId(0), 60000000, env.PDiskName(1, 0))); // Intersects with notification. - const TDuration _10minutes = TDuration::Minutes(10); - env.CheckPermissionRequest("user", false, true, true, true, _10minutes, TStatus::DISALLOW_TEMP, - MakeAction(TAction::REPLACE_DEVICES, env.GetNodeId(0), _10minutes.MicroSeconds(), env.PDiskName(1, 0))); + const TDuration _10minutes = TDuration::Minutes(10); + env.CheckPermissionRequest("user", false, true, true, true, _10minutes, TStatus::DISALLOW_TEMP, + MakeAction(TAction::REPLACE_DEVICES, env.GetNodeId(0), _10minutes.MicroSeconds(), env.PDiskName(1, 0))); // OK with default duration. env.CheckPermissionRequest("user", false, true, true, true, TStatus::ALLOW, @@ -850,7 +850,7 @@ Y_UNIT_TEST_SUITE(TCmsTest) { MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 60000000, "storage")); } - void TestAvailabilityMode(EAvailabilityMode mode, bool disconnectNodes) + void TestAvailabilityMode(EAvailabilityMode mode, bool disconnectNodes) { Y_VERIFY(mode == MODE_KEEP_AVAILABLE || mode == MODE_FORCE_RESTART); @@ -861,62 +861,62 @@ Y_UNIT_TEST_SUITE(TCmsTest) { (env.CheckPermissionRequest("user", false, false, false, true, mode, TStatus::ALLOW, MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(0), 60000000))); - if (disconnectNodes) { - TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = false; - } - + if (disconnectNodes) { + TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = false; + } + env.CheckPermissionRequest("user", false, false, false, - true, mode, TStatus::ALLOW, + true, mode, TStatus::ALLOW, MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(1), 60000000)); - if (disconnectNodes) { - TFakeNodeWhiteboardService::Info[env.GetNodeId(1)].Connected = false; - } - - env.CheckPermissionRequest("user", false, false, false, - true, mode, mode == MODE_KEEP_AVAILABLE ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, - MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(2), 60000000)); - if (mode != MODE_KEEP_AVAILABLE) { - return; - } - + if (disconnectNodes) { + TFakeNodeWhiteboardService::Info[env.GetNodeId(1)].Connected = false; + } + + env.CheckPermissionRequest("user", false, false, false, + true, mode, mode == MODE_KEEP_AVAILABLE ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(2), 60000000)); + if (mode != MODE_KEEP_AVAILABLE) { + return; + } + env.CheckDonePermission("user", res1.second[0]); env.CheckPermissionRequest("user", false, false, false, - true, mode, disconnectNodes ? TStatus::DISALLOW_TEMP: TStatus::ALLOW, - MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(3), 60000000)); - if (!disconnectNodes) { - return; - } + true, mode, disconnectNodes ? TStatus::DISALLOW_TEMP: TStatus::ALLOW, + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(3), 60000000)); + if (!disconnectNodes) { + return; + } - TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = true; + TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = true; env.CheckPermissionRequest("user", false, false, false, - true, mode, TStatus::ALLOW, - MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(3), 60000000)); + true, mode, TStatus::ALLOW, + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(3), 60000000)); } Y_UNIT_TEST(TestKeepAvailableMode) { - TestAvailabilityMode(MODE_KEEP_AVAILABLE, false); + TestAvailabilityMode(MODE_KEEP_AVAILABLE, false); } Y_UNIT_TEST(TestForceRestartMode) { - TestAvailabilityMode(MODE_FORCE_RESTART, false); + TestAvailabilityMode(MODE_FORCE_RESTART, false); } - Y_UNIT_TEST(TestKeepAvailableModeDisconnects) + Y_UNIT_TEST(TestKeepAvailableModeDisconnects) + { + TestAvailabilityMode(MODE_KEEP_AVAILABLE, true); + } + + Y_UNIT_TEST(TestForceRestartModeDisconnects) + { + TestAvailabilityMode(MODE_FORCE_RESTART, true); + } + + void TestAvailabilityModeScheduled(EAvailabilityMode mode, bool disconnectNodes) { - TestAvailabilityMode(MODE_KEEP_AVAILABLE, true); - } - - Y_UNIT_TEST(TestForceRestartModeDisconnects) - { - TestAvailabilityMode(MODE_FORCE_RESTART, true); - } - - void TestAvailabilityModeScheduled(EAvailabilityMode mode, bool disconnectNodes) - { Y_VERIFY(mode == MODE_KEEP_AVAILABLE || mode == MODE_FORCE_RESTART); @@ -928,56 +928,56 @@ Y_UNIT_TEST_SUITE(TCmsTest) { MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(0), 60000000), MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(1), 60000000), MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(2), 60000000))); - if (disconnectNodes) { - TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = false; - } - - env.CheckRequest("user", res1.first, false, mode, TStatus::ALLOW_PARTIAL, 1); - if (disconnectNodes) { - TFakeNodeWhiteboardService::Info[env.GetNodeId(1)].Connected = false; - } - - env.CheckRequest("user", res1.first, false, mode, - mode == MODE_KEEP_AVAILABLE ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, - mode == MODE_KEEP_AVAILABLE ? 0 : 1); - if (mode != MODE_KEEP_AVAILABLE) { - return; - } - - env.CheckDonePermission("user", res1.second[0]); - - env.CheckRequest("user", res1.first, false, mode, - disconnectNodes ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, - disconnectNodes ? 0 : 1); - if (!disconnectNodes) { - return; - } - - TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = true; - - env.CheckRequest("user", res1.first, false, mode, TStatus::ALLOW, 1); + if (disconnectNodes) { + TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = false; + } + + env.CheckRequest("user", res1.first, false, mode, TStatus::ALLOW_PARTIAL, 1); + if (disconnectNodes) { + TFakeNodeWhiteboardService::Info[env.GetNodeId(1)].Connected = false; + } + + env.CheckRequest("user", res1.first, false, mode, + mode == MODE_KEEP_AVAILABLE ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, + mode == MODE_KEEP_AVAILABLE ? 0 : 1); + if (mode != MODE_KEEP_AVAILABLE) { + return; + } + + env.CheckDonePermission("user", res1.second[0]); + + env.CheckRequest("user", res1.first, false, mode, + disconnectNodes ? TStatus::DISALLOW_TEMP : TStatus::ALLOW, + disconnectNodes ? 0 : 1); + if (!disconnectNodes) { + return; + } + + TFakeNodeWhiteboardService::Info[env.GetNodeId(0)].Connected = true; + + env.CheckRequest("user", res1.first, false, mode, TStatus::ALLOW, 1); } Y_UNIT_TEST(TestKeepAvailableModeScheduled) { - TestAvailabilityModeScheduled(MODE_KEEP_AVAILABLE, false); + TestAvailabilityModeScheduled(MODE_KEEP_AVAILABLE, false); } Y_UNIT_TEST(TestForceRestartModeScheduled) { - TestAvailabilityModeScheduled(MODE_FORCE_RESTART, false); + TestAvailabilityModeScheduled(MODE_FORCE_RESTART, false); + } + + Y_UNIT_TEST(TestKeepAvailableModeScheduledDisconnects) + { + TestAvailabilityModeScheduled(MODE_KEEP_AVAILABLE, true); + } + + Y_UNIT_TEST(TestForceRestartModeScheduledDisconnects) + { + TestAvailabilityModeScheduled(MODE_FORCE_RESTART, true); } - Y_UNIT_TEST(TestKeepAvailableModeScheduledDisconnects) - { - TestAvailabilityModeScheduled(MODE_KEEP_AVAILABLE, true); - } - - Y_UNIT_TEST(TestForceRestartModeScheduledDisconnects) - { - TestAvailabilityModeScheduled(MODE_FORCE_RESTART, true); - } - Y_UNIT_TEST(TestOutdatedState) { TCmsTestEnv env(8); @@ -1007,14 +1007,14 @@ Y_UNIT_TEST_SUITE(TCmsTest) { { TCmsTestEnv env(8, 4); - env.CheckSetMarker(MARKER_DISK_BROKEN, "", TStatus::ERROR, + env.CheckSetMarker(MARKER_DISK_BROKEN, "", TStatus::ERROR, "::1", env.PDiskId(0, 0), env.PDiskId(1, 1), env.PDiskId(2, 2), env.PDiskId(3, 3)); - env.CheckResetMarker(MARKER_DISK_BROKEN, "", TStatus::ERROR, + env.CheckResetMarker(MARKER_DISK_BROKEN, "", TStatus::ERROR, "::1", env.PDiskId(0, 0), env.PDiskId(1, 1)); diff --git a/ydb/core/cms/cms_ut_common.cpp b/ydb/core/cms/cms_ut_common.cpp index bc2bc6233d3..b458fe672ce 100644 --- a/ydb/core/cms/cms_ut_common.cpp +++ b/ydb/core/cms/cms_ut_common.cpp @@ -54,7 +54,7 @@ void TFakeNodeWhiteboardService::Handle(TEvBlobStorage::TEvControllerConfigReque driveStatus.SetStatus(NKikimrBlobStorage::ACTIVE); } else if (rec.GetRequest().CommandSize() && rec.GetRequest().GetCommand(0).HasUpdateDriveStatus()) { resp->Record.MutableResponse()->AddStatus()->SetSuccess(true); - resp->Record.MutableResponse()->SetSuccess(true); + resp->Record.MutableResponse()->SetSuccess(true); } ctx.Send(ev->Sender, resp, 0, ev->Cookie); } @@ -212,7 +212,7 @@ public: }; void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseConfig *config, - ui32 pdisks, ui32 vdiskPerPdisk = 4, const TNodeTenantsMap &tenants = {}) + ui32 pdisks, ui32 vdiskPerPdisk = 4, const TNodeTenantsMap &tenants = {}) { TGuard<TMutex> guard(TFakeNodeWhiteboardService::Mutex); ui32 numNodes = runtime.GetNodeCount(); @@ -244,17 +244,17 @@ void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseC node.SystemStateInfo.SetStartTime(now.GetValue()); node.SystemStateInfo.SetChangeTime(now.GetValue()); - if (tenants.contains(nodeIndex)) { - node.SystemStateInfo.AddRoles("Tenant"); - continue; - } else { - node.SystemStateInfo.AddRoles("Storage"); - } - - ui32 groupShift = (nodeIndex / 8) * pdisks * vdiskPerPdisk; - if (numNodes < 8) - groupShift = nodeIndex * numNodeGroups; - + if (tenants.contains(nodeIndex)) { + node.SystemStateInfo.AddRoles("Tenant"); + continue; + } else { + node.SystemStateInfo.AddRoles("Storage"); + } + + ui32 groupShift = (nodeIndex / 8) * pdisks * vdiskPerPdisk; + if (numNodes < 8) + groupShift = nodeIndex * numNodeGroups; + for (ui32 pdiskIndex = 0; pdiskIndex < pdisks; ++pdiskIndex) { auto pdiskId = nodeId * pdisks + pdiskIndex; auto &pdisk = node.PDiskStateInfo[pdiskId]; @@ -272,7 +272,7 @@ void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseC pdiskConfig.SetPDiskId(pdiskId); pdiskConfig.SetPath("/pdisk.data"); pdiskConfig.SetGuid(1); - pdiskConfig.SetDriveStatus(NKikimrBlobStorage::ACTIVE); + pdiskConfig.SetDriveStatus(NKikimrBlobStorage::ACTIVE); for (ui8 vdiskIndex = 0; vdiskIndex < vdiskPerPdisk; ++vdiskIndex) { ui32 vdiskId = pdiskIndex * vdiskPerPdisk + vdiskIndex; @@ -416,11 +416,11 @@ static void SetupServices(TTestActorRuntime &runtime, runtime.Initialize(app.Unwrap()); - auto dnsConfig = new TDynamicNameserviceConfig(); - dnsConfig->MaxStaticNodeId = 1000; - dnsConfig->MaxDynamicNodeId = 2000; - runtime.GetAppData().DynamicNameserviceConfig = dnsConfig; - + auto dnsConfig = new TDynamicNameserviceConfig(); + dnsConfig->MaxStaticNodeId = 1000; + dnsConfig->MaxDynamicNodeId = 2000; + runtime.GetAppData().DynamicNameserviceConfig = dnsConfig; + if (!runtime.IsRealThreads()) { TDispatchOptions options; options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvBlobStorage::EvLocalRecoveryDone, @@ -449,7 +449,7 @@ TCmsTestEnv::TCmsTestEnv(ui32 nodeCount, status.SetSuccess(true); auto *config = status.MutableBaseConfig(); - GenerateExtendedInfo(*this, config, pdisks, 4, tenants); + GenerateExtendedInfo(*this, config, pdisks, 4, tenants); // Set observer to pass fake base blobstorage config. auto redirectConfigRequest = [](TTestActorRuntimeBase&, @@ -479,7 +479,7 @@ TCmsTestEnv::TCmsTestEnv(ui32 nodeCount, TCmsTestEnv::TCmsTestEnv(ui32 nodeCount, const TNodeTenantsMap &tenants) - : TCmsTestEnv(nodeCount, 0, tenants) + : TCmsTestEnv(nodeCount, 0, tenants) { } diff --git a/ydb/core/cms/config.h b/ydb/core/cms/config.h index a3d013c7f11..471880e34f6 100644 --- a/ydb/core/cms/config.h +++ b/ydb/core/cms/config.h @@ -1,120 +1,120 @@ #pragma once -#include "pdisk_state.h" - +#include "pdisk_state.h" + #include <ydb/core/protos/cms.pb.h> #include <util/datetime/base.h> #include <util/generic/hash.h> -#include <util/generic/map.h> +#include <util/generic/map.h> namespace NKikimr { namespace NCms { -struct TCmsSentinelConfig { - bool Enable = false; - bool DryRun = false; - - TDuration UpdateConfigInterval; - TDuration RetryUpdateConfig; - - TDuration UpdateStateInterval; - TDuration UpdateStateTimeout; - - TDuration RetryChangeStatus; - ui32 ChangeStatusRetries; - - ui32 DefaultStateLimit; - TMap<EPDiskState, ui32> StateLimits; - - ui32 DataCenterRatio; - ui32 RoomRatio; - ui32 RackRatio; - - void Serialize(NKikimrCms::TCmsConfig::TSentinelConfig &config) const - { - config.SetEnable(Enable); - config.SetDryRun(DryRun); - config.SetUpdateConfigInterval(UpdateConfigInterval.GetValue()); - config.SetRetryUpdateConfig(RetryUpdateConfig.GetValue()); - config.SetUpdateStateInterval(UpdateStateInterval.GetValue()); - config.SetUpdateStateTimeout(UpdateStateTimeout.GetValue()); - config.SetRetryChangeStatus(RetryChangeStatus.GetValue()); - config.SetChangeStatusRetries(ChangeStatusRetries); - config.SetDefaultStateLimit(DefaultStateLimit); - config.SetDataCenterRatio(DataCenterRatio); - config.SetRoomRatio(RoomRatio); - config.SetRackRatio(RackRatio); - - SaveStateLimits(config); - } - - void Deserialize(const NKikimrCms::TCmsConfig::TSentinelConfig &config) - { - Enable = config.GetEnable(); - DryRun = config.GetDryRun(); - UpdateConfigInterval = TDuration::MicroSeconds(config.GetUpdateConfigInterval()); - RetryUpdateConfig = TDuration::MicroSeconds(config.GetRetryUpdateConfig()); - UpdateStateInterval = TDuration::MicroSeconds(config.GetUpdateStateInterval()); - UpdateStateTimeout = TDuration::MicroSeconds(config.GetUpdateStateTimeout()); - RetryChangeStatus = TDuration::MicroSeconds(config.GetRetryChangeStatus()); - ChangeStatusRetries = config.GetChangeStatusRetries(); - DefaultStateLimit = config.GetDefaultStateLimit(); - DataCenterRatio = config.GetDataCenterRatio(); - RoomRatio = config.GetRoomRatio(); - RackRatio = config.GetRackRatio(); - - auto newStateLimits = LoadStateLimits(config); - StateLimits.swap(newStateLimits); - } - - void SaveStateLimits(NKikimrCms::TCmsConfig::TSentinelConfig &config) const - { - auto defaultStateLimits = DefaultStateLimits(); - bool differsFromDefault = false; - - if (defaultStateLimits.size() != StateLimits.size()) { - differsFromDefault = true; - } else { - for (const auto& [state, limit] : defaultStateLimits) { - auto it = StateLimits.find(state); - if (it == StateLimits.end() || limit != it->second) { - differsFromDefault = true; - break; - } - } - } - - if (!differsFromDefault) { - return; - } - - for (const auto& [state, limit] : StateLimits) { - auto& stateLimit = *config.AddStateLimits(); - stateLimit.SetState(static_cast<ui32>(state)); - stateLimit.SetLimit(limit); - } - } - - static TMap<EPDiskState, ui32> LoadStateLimits(const NKikimrCms::TCmsConfig::TSentinelConfig &config) - { - TMap<EPDiskState, ui32> stateLimits; - - for (const auto &val : config.GetStateLimits()) { - stateLimits[static_cast<EPDiskState>(val.GetState())] = val.GetLimit(); - } - - if (stateLimits) { - return stateLimits; - } - - return DefaultStateLimits(); - } - - static TMap<EPDiskState, ui32> DefaultStateLimits() - { - TMap<EPDiskState, ui32> stateLimits; - // error states +struct TCmsSentinelConfig { + bool Enable = false; + bool DryRun = false; + + TDuration UpdateConfigInterval; + TDuration RetryUpdateConfig; + + TDuration UpdateStateInterval; + TDuration UpdateStateTimeout; + + TDuration RetryChangeStatus; + ui32 ChangeStatusRetries; + + ui32 DefaultStateLimit; + TMap<EPDiskState, ui32> StateLimits; + + ui32 DataCenterRatio; + ui32 RoomRatio; + ui32 RackRatio; + + void Serialize(NKikimrCms::TCmsConfig::TSentinelConfig &config) const + { + config.SetEnable(Enable); + config.SetDryRun(DryRun); + config.SetUpdateConfigInterval(UpdateConfigInterval.GetValue()); + config.SetRetryUpdateConfig(RetryUpdateConfig.GetValue()); + config.SetUpdateStateInterval(UpdateStateInterval.GetValue()); + config.SetUpdateStateTimeout(UpdateStateTimeout.GetValue()); + config.SetRetryChangeStatus(RetryChangeStatus.GetValue()); + config.SetChangeStatusRetries(ChangeStatusRetries); + config.SetDefaultStateLimit(DefaultStateLimit); + config.SetDataCenterRatio(DataCenterRatio); + config.SetRoomRatio(RoomRatio); + config.SetRackRatio(RackRatio); + + SaveStateLimits(config); + } + + void Deserialize(const NKikimrCms::TCmsConfig::TSentinelConfig &config) + { + Enable = config.GetEnable(); + DryRun = config.GetDryRun(); + UpdateConfigInterval = TDuration::MicroSeconds(config.GetUpdateConfigInterval()); + RetryUpdateConfig = TDuration::MicroSeconds(config.GetRetryUpdateConfig()); + UpdateStateInterval = TDuration::MicroSeconds(config.GetUpdateStateInterval()); + UpdateStateTimeout = TDuration::MicroSeconds(config.GetUpdateStateTimeout()); + RetryChangeStatus = TDuration::MicroSeconds(config.GetRetryChangeStatus()); + ChangeStatusRetries = config.GetChangeStatusRetries(); + DefaultStateLimit = config.GetDefaultStateLimit(); + DataCenterRatio = config.GetDataCenterRatio(); + RoomRatio = config.GetRoomRatio(); + RackRatio = config.GetRackRatio(); + + auto newStateLimits = LoadStateLimits(config); + StateLimits.swap(newStateLimits); + } + + void SaveStateLimits(NKikimrCms::TCmsConfig::TSentinelConfig &config) const + { + auto defaultStateLimits = DefaultStateLimits(); + bool differsFromDefault = false; + + if (defaultStateLimits.size() != StateLimits.size()) { + differsFromDefault = true; + } else { + for (const auto& [state, limit] : defaultStateLimits) { + auto it = StateLimits.find(state); + if (it == StateLimits.end() || limit != it->second) { + differsFromDefault = true; + break; + } + } + } + + if (!differsFromDefault) { + return; + } + + for (const auto& [state, limit] : StateLimits) { + auto& stateLimit = *config.AddStateLimits(); + stateLimit.SetState(static_cast<ui32>(state)); + stateLimit.SetLimit(limit); + } + } + + static TMap<EPDiskState, ui32> LoadStateLimits(const NKikimrCms::TCmsConfig::TSentinelConfig &config) + { + TMap<EPDiskState, ui32> stateLimits; + + for (const auto &val : config.GetStateLimits()) { + stateLimits[static_cast<EPDiskState>(val.GetState())] = val.GetLimit(); + } + + if (stateLimits) { + return stateLimits; + } + + return DefaultStateLimits(); + } + + static TMap<EPDiskState, ui32> DefaultStateLimits() + { + TMap<EPDiskState, ui32> stateLimits; + // error states stateLimits[NKikimrBlobStorage::TPDiskState::InitialFormatReadError] = 60; stateLimits[NKikimrBlobStorage::TPDiskState::InitialSysLogReadError] = 60; stateLimits[NKikimrBlobStorage::TPDiskState::InitialSysLogParseError] = 60; @@ -124,19 +124,19 @@ struct TCmsSentinelConfig { stateLimits[NKikimrBlobStorage::TPDiskState::OpenFileError] = 60; stateLimits[NKikimrBlobStorage::TPDiskState::ChunkQuotaError] = 60; stateLimits[NKikimrBlobStorage::TPDiskState::DeviceIoError] = 60; - // node online, pdisk missing + // node online, pdisk missing stateLimits[NKikimrBlobStorage::TPDiskState::Missing] = 60; - // node timeout + // node timeout stateLimits[NKikimrBlobStorage::TPDiskState::Timeout] = 60; - // node offline + // node offline stateLimits[NKikimrBlobStorage::TPDiskState::NodeDisconnected] = 60; - // disable for unknown states + // disable for unknown states stateLimits[NKikimrBlobStorage::TPDiskState::Unknown] = 0; - - return stateLimits; - } -}; - + + return stateLimits; + } +}; + struct TCmsLogConfig { THashMap<ui32, bool> RecordLevels; bool EnabledByDefault = true; @@ -185,7 +185,7 @@ struct TCmsConfig { TDuration InfoCollectionTimeout; NKikimrCms::TLimits TenantLimits; NKikimrCms::TLimits ClusterLimits; - TCmsSentinelConfig SentinelConfig; + TCmsSentinelConfig SentinelConfig; TCmsLogConfig LogConfig; TCmsConfig() @@ -202,10 +202,10 @@ struct TCmsConfig { { config.SetDefaultRetryTime(DefaultRetryTime.GetValue()); config.SetDefaultPermissionDuration(DefaultPermissionDuration.GetValue()); - config.SetInfoCollectionTimeout(InfoCollectionTimeout.GetValue()); + config.SetInfoCollectionTimeout(InfoCollectionTimeout.GetValue()); config.MutableTenantLimits()->CopyFrom(TenantLimits); config.MutableClusterLimits()->CopyFrom(ClusterLimits); - SentinelConfig.Serialize(*config.MutableSentinelConfig()); + SentinelConfig.Serialize(*config.MutableSentinelConfig()); LogConfig.Serialize(*config.MutableLogConfig()); } @@ -213,10 +213,10 @@ struct TCmsConfig { { DefaultRetryTime = TDuration::MicroSeconds(config.GetDefaultRetryTime()); DefaultPermissionDuration = TDuration::MicroSeconds(config.GetDefaultPermissionDuration()); - InfoCollectionTimeout = TDuration::MicroSeconds(config.GetInfoCollectionTimeout()); + InfoCollectionTimeout = TDuration::MicroSeconds(config.GetInfoCollectionTimeout()); TenantLimits.CopyFrom(config.GetTenantLimits()); ClusterLimits.CopyFrom(config.GetClusterLimits()); - SentinelConfig.Deserialize(config.GetSentinelConfig()); + SentinelConfig.Deserialize(config.GetSentinelConfig()); LogConfig.Deserialize(config.GetLogConfig()); } diff --git a/ydb/core/cms/console/configs_dispatcher_ut.cpp b/ydb/core/cms/console/configs_dispatcher_ut.cpp index 52934463d81..997d1ba4514 100644 --- a/ydb/core/cms/console/configs_dispatcher_ut.cpp +++ b/ydb/core/cms/console/configs_dispatcher_ut.cpp @@ -67,15 +67,15 @@ TActorId InitConfigsDispatcher(TTenantTestRuntime &runtime) NKikimrConfig::TAppConfig(), {}, {}, "", "", 2, NKikimrConsole::TConfigItem::MERGE, ""); - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME) - .WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME) + .WithPools({{"hdd", 1}})); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, Ydb::Cms::GetDatabaseStatusResult::PENDING_RESOURCES, {{"hdd", 1, 1}}, {}); - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithPools({{"hdd", 1}})); CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, Ydb::Cms::GetDatabaseStatusResult::PENDING_RESOURCES, {{"hdd", 1, 1}}, {}); diff --git a/ydb/core/cms/console/console__alter_tenant.cpp b/ydb/core/cms/console/console__alter_tenant.cpp index b24ee5fb689..dc44c250828 100644 --- a/ydb/core/cms/console/console__alter_tenant.cpp +++ b/ydb/core/cms/console/console__alter_tenant.cpp @@ -119,12 +119,12 @@ public: // Check added storage resource units. ui64 newGroups = 0; for (auto &unit : rec.storage_units_to_add()) { - auto &kind = unit.unit_kind(); - - if (Tenant->AreResourcesShared && !Tenant->StoragePools.contains(kind)) - return Error(Ydb::StatusIds::UNSUPPORTED, - Sprintf("Database '%s' is shared, cannot add new storage units", path.data()), ctx); - + auto &kind = unit.unit_kind(); + + if (Tenant->AreResourcesShared && !Tenant->StoragePools.contains(kind)) + return Error(Ydb::StatusIds::UNSUPPORTED, + Sprintf("Database '%s' is shared, cannot add new storage units", path.data()), ctx); + if (Tenant->SharedDomainId) return Error(Ydb::StatusIds::BAD_REQUEST, Sprintf("Database '%s' is serverless, cannot add storage units", path.data()), ctx); @@ -239,7 +239,7 @@ public: pool->AddRequiredGroups(size); pool->State = TStoragePool::NOT_UPDATED; } else { - Y_VERIFY(!Tenant->AreResourcesShared); + Y_VERIFY(!Tenant->AreResourcesShared); pool = Self->MakeStoragePool(Tenant, kind, size); } diff --git a/ydb/core/cms/console/console__configure.cpp b/ydb/core/cms/console/console__configure.cpp index 74de0b4429b..f28417e7600 100644 --- a/ydb/core/cms/console/console__configure.cpp +++ b/ydb/core/cms/console/console__configure.cpp @@ -405,25 +405,25 @@ public: return true; } - TDynBitMap GetAffectedKinds(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigureAction>& actions) { - TDynBitMap kinds; - - for (auto &action : actions) { - switch (action.GetActionCase()) { - case NKikimrConsole::TConfigureAction::kAddConfigItem: - kinds.Set(action.GetAddConfigItem().GetConfigItem().GetKind()); - break; - case NKikimrConsole::TConfigureAction::kModifyConfigItem: - kinds.Set(action.GetModifyConfigItem().GetConfigItem().GetKind()); - break; - default: - return TDynBitMap(); - } - } - - return kinds; - } - + TDynBitMap GetAffectedKinds(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigureAction>& actions) { + TDynBitMap kinds; + + for (auto &action : actions) { + switch (action.GetActionCase()) { + case NKikimrConsole::TConfigureAction::kAddConfigItem: + kinds.Set(action.GetAddConfigItem().GetConfigItem().GetKind()); + break; + case NKikimrConsole::TConfigureAction::kModifyConfigItem: + kinds.Set(action.GetModifyConfigItem().GetConfigItem().GetKind()); + break; + default: + return TDynBitMap(); + } + } + + return kinds; + } + bool ProcessAutoOrdering(const TActorContext &ctx) { for (auto &item : OrderingQueue) @@ -492,7 +492,7 @@ public: TModificationsValidator affectedChecker(Self->ConfigIndex, Self->PendingConfigModifications, config); - auto affected = affectedChecker.ComputeAffectedConfigs(GetAffectedKinds(rec.GetActions()), false); + auto affected = affectedChecker.ComputeAffectedConfigs(GetAffectedKinds(rec.GetActions()), false); LOG_DEBUG_S(ctx, NKikimrServices::CMS_CONFIGS, "affected.size() = " << affected.size()); for (auto &item : affected) { diff --git a/ydb/core/cms/console/console__create_tenant.cpp b/ydb/core/cms/console/console__create_tenant.cpp index ddd2783eabf..77d8166d547 100644 --- a/ydb/core/cms/console/console__create_tenant.cpp +++ b/ydb/core/cms/console/console__create_tenant.cpp @@ -3,20 +3,20 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; class TTenantsManager::TTxCreateTenant : public TTransactionBase<TTenantsManager> { - static const Ydb::Cms::Resources& GetResources(const Ydb::Cms::CreateDatabaseRequest& request) { - switch (request.resources_kind_case()) { - case Ydb::Cms::CreateDatabaseRequest::kResources: - return request.resources(); - case Ydb::Cms::CreateDatabaseRequest::kSharedResources: - return request.shared_resources(); - default: - Y_FAIL_S("There is no resources: " << static_cast<ui32>(request.resources_kind_case())); - } - } - + static const Ydb::Cms::Resources& GetResources(const Ydb::Cms::CreateDatabaseRequest& request) { + switch (request.resources_kind_case()) { + case Ydb::Cms::CreateDatabaseRequest::kResources: + return request.resources(); + case Ydb::Cms::CreateDatabaseRequest::kSharedResources: + return request.shared_resources(); + default: + Y_FAIL_S("There is no resources: " << static_cast<ui32>(request.resources_kind_case())); + } + } + public: TTxCreateTenant(TEvConsole::TEvCreateTenantRequest::TPtr ev, TTenantsManager *self) : TBase(self) @@ -79,17 +79,17 @@ public: return Error(Ydb::StatusIds::BAD_REQUEST, error, ctx); path = CanonizePath(path); - if (auto tenant = Self->GetTenant(path)) { + if (auto tenant = Self->GetTenant(path)) { if (rec.idempotency_key() && tenant->CreateIdempotencyKey == rec.idempotency_key()) { return Pending(tenant); } else if (tenant->IsRemoving()) { - return Error(Ydb::StatusIds::PRECONDITION_FAILED, - Sprintf("Database '%s' is removing", path.data()), ctx); - } else { - return Error(Ydb::StatusIds::ALREADY_EXISTS, - Sprintf("Database '%s' already exists", path.data()), ctx); - } - } + return Error(Ydb::StatusIds::PRECONDITION_FAILED, + Sprintf("Database '%s' is removing", path.data()), ctx); + } else { + return Error(Ydb::StatusIds::ALREADY_EXISTS, + Sprintf("Database '%s' already exists", path.data()), ctx); + } + } if (auto it = Self->RemovedTenants.find(path); it != Self->RemovedTenants.end()) { if (rec.idempotency_key() && it->second.CreateIdempotencyKey == rec.idempotency_key()) { @@ -123,7 +123,7 @@ public: } Tenant = new TTenant(path, TTenant::CREATING_POOLS, token); - + Tenant->IsExternalSubdomain = Self->FeatureFlags.GetEnableExternalSubdomains(); Tenant->IsExternalHive = Self->FeatureFlags.GetEnableExternalHive(); Tenant->IsExternalSysViewProcessor = Self->FeatureFlags.GetEnablePersistentQueryStats(); @@ -145,118 +145,118 @@ public: Tenant->IsExternalHive = false; Tenant->IsExternalSysViewProcessor = false; } - + Tenant->IsExternalHive &= Tenant->IsExternalSubdomain; // external hive without external sub domain is pointless Tenant->IsExternalSysViewProcessor &= Tenant->IsExternalSubdomain; - + Tenant->StorageUnitsQuota = Self->Config.DefaultStorageUnitsQuota; Tenant->ComputationalUnitsQuota = Self->Config.DefaultComputationalUnitsQuota; - + for (const auto& [key, value] : rec.attributes()) { auto &r = *Tenant->Attributes.AddUserAttributes(); r.SetKey(key); r.SetValue(value); } - switch (rec.resources_kind_case()) { - case Ydb::Cms::CreateDatabaseRequest::kSharedResources: - if (rec.options().disable_tx_service()) { - return Error(Ydb::StatusIds::BAD_REQUEST, - "Cannot create shared database with no tx service enabled", ctx); - } + switch (rec.resources_kind_case()) { + case Ydb::Cms::CreateDatabaseRequest::kSharedResources: + if (rec.options().disable_tx_service()) { + return Error(Ydb::StatusIds::BAD_REQUEST, + "Cannot create shared database with no tx service enabled", ctx); + } - Tenant->AreResourcesShared = true; + Tenant->AreResourcesShared = true; // fallthrough to normal resources handling [[fallthrough]]; - case Ydb::Cms::CreateDatabaseRequest::kResources: - for (auto &unit : GetResources(rec).storage_units()) { - auto &kind = unit.unit_kind(); - auto size = unit.count(); - - if (Tenant->StoragePools.contains(kind)) { - auto pool = Tenant->StoragePools.at(kind); - pool->AddRequiredGroups(size); - } else { - if (!Self->MakeBasicPoolCheck(kind, size, code, error)) - return Error(code, error, ctx); - - auto poolName = Tenant->MakeStoragePoolName(kind); - auto &config = Self->Domain->StoragePoolTypes.at(kind); - TStoragePool::TPtr pool = new TStoragePool(poolName, kind, size, config); - Tenant->StoragePools.emplace(std::make_pair(kind, pool)); - } + case Ydb::Cms::CreateDatabaseRequest::kResources: + for (auto &unit : GetResources(rec).storage_units()) { + auto &kind = unit.unit_kind(); + auto size = unit.count(); + + if (Tenant->StoragePools.contains(kind)) { + auto pool = Tenant->StoragePools.at(kind); + pool->AddRequiredGroups(size); + } else { + if (!Self->MakeBasicPoolCheck(kind, size, code, error)) + return Error(code, error, ctx); + + auto poolName = Tenant->MakeStoragePoolName(kind); + auto &config = Self->Domain->StoragePoolTypes.at(kind); + TStoragePool::TPtr pool = new TStoragePool(poolName, kind, size, config); + Tenant->StoragePools.emplace(std::make_pair(kind, pool)); + } } - if (Tenant->StoragePools.empty()) - return Error(Ydb::StatusIds::BAD_REQUEST, - "No storage units specified.", ctx); - - for (auto &unit : GetResources(rec).computational_units()) { - auto &kind = unit.unit_kind(); - auto &zone = unit.availability_zone(); - ui64 count = unit.count(); - - if (!Self->MakeBasicComputationalUnitCheck(kind, zone, code, error)) - return Error(code, error, ctx); - - Tenant->ComputationalUnits[std::make_pair(kind, zone)] += count; - } - - // Check tenant quotas. - if (!Tenant->CheckQuota(code, error)) + if (Tenant->StoragePools.empty()) + return Error(Ydb::StatusIds::BAD_REQUEST, + "No storage units specified.", ctx); + + for (auto &unit : GetResources(rec).computational_units()) { + auto &kind = unit.unit_kind(); + auto &zone = unit.availability_zone(); + ui64 count = unit.count(); + + if (!Self->MakeBasicComputationalUnitCheck(kind, zone, code, error)) + return Error(code, error, ctx); + + Tenant->ComputationalUnits[std::make_pair(kind, zone)] += count; + } + + // Check tenant quotas. + if (!Tenant->CheckQuota(code, error)) + return Error(code, error, ctx); + + // Check cluster quotas. + if (!Self->CheckComputationalUnitsQuota(Tenant->ComputationalUnits, code, error)) return Error(code, error, ctx); - // Check cluster quotas. - if (!Self->CheckComputationalUnitsQuota(Tenant->ComputationalUnits, code, error)) - return Error(code, error, ctx); - - Tenant->ParseComputationalUnits(Self->Config); - break; - - case Ydb::Cms::CreateDatabaseRequest::kServerlessResources: - if (!Tenant->IsExternalSubdomain) { - return Error(Ydb::StatusIds::PRECONDITION_FAILED, - "Cannot create serverless database unless external subdomain is enabled", ctx); - } - - if (rec.options().disable_tx_service()) { - return Error(Ydb::StatusIds::BAD_REQUEST, - "Cannot create serverless database with no tx service enabled", ctx); - } - - if (const TString& sharedDbPath = rec.serverless_resources().shared_database_path()) { - if (auto tenant = Self->GetTenant(CanonizePath(sharedDbPath))) { - if (!tenant->AreResourcesShared) { - return Error(Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Database is not shared: " << sharedDbPath, ctx); - } - - if (!tenant->IsRunning()) { - return Error(Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Database is not running: " << sharedDbPath, ctx); - } - - Y_VERIFY(tenant->DomainId); - Tenant->SharedDomainId = tenant->DomainId; - tenant->HostedTenants.emplace(Tenant); - - Tenant->IsExternalHive = false; - Tenant->Coordinators = 1; - Tenant->SlotsAllocationConfirmed = true; - } else { - return Error(Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Database not exists: " << sharedDbPath, ctx); - } - } else { - return Error(Ydb::StatusIds::BAD_REQUEST, "Empty database name", ctx); - } - break; - - default: - return Error(Ydb::StatusIds::BAD_REQUEST, "Unknown resources kind", ctx); - } - + Tenant->ParseComputationalUnits(Self->Config); + break; + + case Ydb::Cms::CreateDatabaseRequest::kServerlessResources: + if (!Tenant->IsExternalSubdomain) { + return Error(Ydb::StatusIds::PRECONDITION_FAILED, + "Cannot create serverless database unless external subdomain is enabled", ctx); + } + + if (rec.options().disable_tx_service()) { + return Error(Ydb::StatusIds::BAD_REQUEST, + "Cannot create serverless database with no tx service enabled", ctx); + } + + if (const TString& sharedDbPath = rec.serverless_resources().shared_database_path()) { + if (auto tenant = Self->GetTenant(CanonizePath(sharedDbPath))) { + if (!tenant->AreResourcesShared) { + return Error(Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Database is not shared: " << sharedDbPath, ctx); + } + + if (!tenant->IsRunning()) { + return Error(Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Database is not running: " << sharedDbPath, ctx); + } + + Y_VERIFY(tenant->DomainId); + Tenant->SharedDomainId = tenant->DomainId; + tenant->HostedTenants.emplace(Tenant); + + Tenant->IsExternalHive = false; + Tenant->Coordinators = 1; + Tenant->SlotsAllocationConfirmed = true; + } else { + return Error(Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Database not exists: " << sharedDbPath, ctx); + } + } else { + return Error(Ydb::StatusIds::BAD_REQUEST, "Empty database name", ctx); + } + break; + + default: + return Error(Ydb::StatusIds::BAD_REQUEST, "Unknown resources kind", ctx); + } + if (rec.has_schema_operation_quotas()) { Tenant->SchemaOperationQuotas.ConstructInPlace(rec.schema_operation_quotas()); } diff --git a/ydb/core/cms/console/console__remove_computational_units.cpp b/ydb/core/cms/console/console__remove_computational_units.cpp index e033e4437d1..699c1b5c2af 100644 --- a/ydb/core/cms/console/console__remove_computational_units.cpp +++ b/ydb/core/cms/console/console__remove_computational_units.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; class TTenantsManager::TTxRemoveComputationalUnits : public TTransactionBase<TTenantsManager> { public: diff --git a/ydb/core/cms/console/console__remove_tenant.cpp b/ydb/core/cms/console/console__remove_tenant.cpp index 952bc127f2f..14bf4e19f36 100644 --- a/ydb/core/cms/console/console__remove_tenant.cpp +++ b/ydb/core/cms/console/console__remove_tenant.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; class TTenantsManager::TTxRemoveTenant : public TTransactionBase<TTenantsManager> { public: @@ -70,9 +70,9 @@ public: } else if (!Tenant->IsConfiguring() && !Tenant->IsRunning()) { return Error(Ydb::StatusIds::UNAVAILABLE, Sprintf("Database '%s' is busy", path.data()), ctx); - } else if (Tenant->HostedTenants) { - return Error(Ydb::StatusIds::PRECONDITION_FAILED, - Sprintf("Database '%s' has serverless databases. Remove all of them first", path.data()), ctx); + } else if (Tenant->HostedTenants) { + return Error(Ydb::StatusIds::PRECONDITION_FAILED, + Sprintf("Database '%s' has serverless databases. Remove all of them first", path.data()), ctx); } Tenant->TxId = ctx.Now().GetValue(); diff --git a/ydb/core/cms/console/console__remove_tenant_done.cpp b/ydb/core/cms/console/console__remove_tenant_done.cpp index b2e10c23db5..406b3d473ae 100644 --- a/ydb/core/cms/console/console__remove_tenant_done.cpp +++ b/ydb/core/cms/console/console__remove_tenant_done.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; class TTenantsManager::TTxRemoveTenantDone : public TTransactionBase<TTenantsManager> { public: diff --git a/ydb/core/cms/console/console__remove_tenant_failed.cpp b/ydb/core/cms/console/console__remove_tenant_failed.cpp index dd0af0df722..541bb5d5936 100644 --- a/ydb/core/cms/console/console__remove_tenant_failed.cpp +++ b/ydb/core/cms/console/console__remove_tenant_failed.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; class TTenantsManager::TTxRemoveTenantFailed : public TTransactionBase<TTenantsManager> { public: diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index b8287bb3219..29eb6c3035a 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -31,19 +31,19 @@ struct Schema : NIceDb::Schema { struct TimeCastBucketsPerMediator : Column<12, NScheme::NTypeIds::Uint32> {}; struct Attributes : Column<13, NScheme::NTypeIds::String> { using Type = NKikimrSchemeOp::TAlterUserAttributes; }; struct Generation : Column<14, NScheme::NTypeIds::Uint64> {}; - // DomainId { + // DomainId { struct SchemeShardId : Column<15, NScheme::NTypeIds::Uint64> {}; struct PathId : Column<16, NScheme::NTypeIds::Uint64> {}; - // } // DomainId + // } // DomainId struct ErrorCode : Column<17, NScheme::NTypeIds::Uint32> {}; struct IsExternalSubDomain : Column<18, NScheme::NTypeIds::Bool> {}; struct IsExternalHive : Column<19, NScheme::NTypeIds::Bool> {}; - struct AreResourcesShared : Column<20, NScheme::NTypeIds::Bool> {}; - // SharedDomainId { - struct SharedDomainSchemeShardId : Column<21, NScheme::NTypeIds::Uint64> {}; - struct SharedDomainPathId : Column<22, NScheme::NTypeIds::Uint64> {}; + struct AreResourcesShared : Column<20, NScheme::NTypeIds::Bool> {}; + // SharedDomainId { + struct SharedDomainSchemeShardId : Column<21, NScheme::NTypeIds::Uint64> {}; + struct SharedDomainPathId : Column<22, NScheme::NTypeIds::Uint64> {}; struct IsExternalSysViewProcessor : Column<23, NScheme::NTypeIds::Bool> {}; - // } // SharedDomainId + // } // SharedDomainId struct SchemaOperationQuotas : Column<24, NScheme::NTypeIds::String> {}; struct CreateIdempotencyKey : Column<25, NScheme::NTypeIds::Utf8> {}; struct AlterIdempotencyKey : Column<26, NScheme::NTypeIds::Utf8> {}; @@ -52,7 +52,7 @@ struct Schema : NIceDb::Schema { using TKey = TableKey<Path>; using TColumns = TableColumns<Path, State, Coordinators, Mediators, PlanResolution, Issue, TxId, UserToken, SubdomainVersion, ConfirmedSubdomain, TimeCastBucketsPerMediator, - Attributes, Generation, SchemeShardId, PathId, ErrorCode, IsExternalSubDomain, IsExternalHive, + Attributes, Generation, SchemeShardId, PathId, ErrorCode, IsExternalSubDomain, IsExternalHive, AreResourcesShared, SharedDomainSchemeShardId, SharedDomainPathId, IsExternalSysViewProcessor, SchemaOperationQuotas, CreateIdempotencyKey, AlterIdempotencyKey, DatabaseQuotas>; }; diff --git a/ydb/core/cms/console/console__update_subdomain_key.cpp b/ydb/core/cms/console/console__update_subdomain_key.cpp index b0cd712e639..6a9fcba8cb7 100644 --- a/ydb/core/cms/console/console__update_subdomain_key.cpp +++ b/ydb/core/cms/console/console__update_subdomain_key.cpp @@ -66,15 +66,15 @@ public: "TTxUpdateSubDomainKey complete for " << Path); if (Tenant) { - if (Tenant->DomainId) { - Self->TenantIdToName.erase(Tenant->DomainId); - } - - Tenant->DomainId = TDomainId(SchemeShardId, PathId); - - Y_VERIFY(!Self->TenantIdToName.contains(Tenant->DomainId)); - Self->TenantIdToName[Tenant->DomainId] = Tenant->Path; - + if (Tenant->DomainId) { + Self->TenantIdToName.erase(Tenant->DomainId); + } + + Tenant->DomainId = TDomainId(SchemeShardId, PathId); + + Y_VERIFY(!Self->TenantIdToName.contains(Tenant->DomainId)); + Self->TenantIdToName[Tenant->DomainId] = Tenant->Path; + if (Tenant->Worker == Worker) Tenant->Worker = TActorId(); diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index 9277e9948b2..0db3f5dceda 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -23,7 +23,7 @@ namespace NKikimr { namespace NConsole { -using namespace NOperationId; +using namespace NOperationId; using namespace NTenantSlotBroker; namespace { @@ -280,9 +280,9 @@ public: } const auto &scope = rec.GetStatus(0).GetStoragePool(0).GetScopeId(); - if (TTenantsManager::TDomainId(scope.GetX1(), scope.GetX2()) != Tenant->DomainId) { + if (TTenantsManager::TDomainId(scope.GetX1(), scope.GetX2()) != Tenant->DomainId) { BLOG_ERROR(LogPrefix << "scope id check failure " - << Tenant->DomainId + << Tenant->DomainId << " vs " << scope.ShortDebugString()); ReplyAndDie(new TTenantsManager::TEvPrivate::TEvPoolFailed(Tenant, Pool, Pool->Issue), ctx); return; @@ -385,9 +385,9 @@ private: TActorId OwnerId; TTenantsManager::TTenant::TPtr Tenant; - TTenantsManager::TTenant::TPtr SharedTenant; + TTenantsManager::TTenant::TPtr SharedTenant; EAction Action; - // Pair of <WorkDir, DirToCreate> + // Pair of <WorkDir, DirToCreate> std::pair<TString, TString> Subdomain; ui64 TxId; ui64 TabletId; @@ -406,10 +406,10 @@ public: } TSubDomainManip(TActorId ownerId, TTenantsManager::TTenant::TPtr tenant, EAction action, - TTenantsManager::TTenant::TPtr sharedTenant = nullptr) + TTenantsManager::TTenant::TPtr sharedTenant = nullptr) : OwnerId(ownerId) , Tenant(tenant) - , SharedTenant(sharedTenant) + , SharedTenant(sharedTenant) , Action(action) , TxId(0) , TabletId(0) @@ -417,25 +417,25 @@ public: , SchemeshardId(0) , PathId(0) { - if (Action == CREATE) { - TVector<TString> parts = SplitPath(Tenant->Path); - TString workDir; - TString pathToCreate; - - for (size_t i = 0; i < parts.size(); ++i) { - if (i == 0) { - workDir = "/" + parts[i]; + if (Action == CREATE) { + TVector<TString> parts = SplitPath(Tenant->Path); + TString workDir; + TString pathToCreate; + + for (size_t i = 0; i < parts.size(); ++i) { + if (i == 0) { + workDir = "/" + parts[i]; } else { - if (pathToCreate) { - pathToCreate += "/"; - } - pathToCreate += parts[i]; + if (pathToCreate) { + pathToCreate += "/"; + } + pathToCreate += parts[i]; } } - - Subdomain = std::make_pair(workDir, pathToCreate); - } else { - Subdomain = std::make_pair(ExtractParent(Tenant->Path), ExtractBase(Tenant->Path)); + + Subdomain = std::make_pair(workDir, pathToCreate); + } else { + Subdomain = std::make_pair(ExtractParent(Tenant->Path), ExtractBase(Tenant->Path)); } if (IssuesMap.empty()) { @@ -459,13 +459,13 @@ public: subdomain.SetExternalSysViewProcessor(true); } } - - if (SharedTenant) { - const auto &resourcesDomainId = SharedTenant->DomainId; - auto &resourcesDomainKey = *subdomain.MutableResourcesDomainKey(); - resourcesDomainKey.SetSchemeShard(resourcesDomainId.OwnerId); - resourcesDomainKey.SetPathId(resourcesDomainId.LocalPathId); - } + + if (SharedTenant) { + const auto &resourcesDomainId = SharedTenant->DomainId; + auto &resourcesDomainKey = *subdomain.MutableResourcesDomainKey(); + resourcesDomainKey.SetSchemeShard(resourcesDomainId.OwnerId); + resourcesDomainKey.SetPathId(resourcesDomainId.LocalPathId); + } } void FillSubdomainAlterInfo(NKikimrSubDomains::TSubDomainSettings &subdomain, @@ -487,8 +487,8 @@ public: subdomain.SetExternalSysViewProcessor(true); } } - - for (auto &pr : (SharedTenant ? SharedTenant->StoragePools : Tenant->StoragePools)) { + + for (auto &pr : (SharedTenant ? SharedTenant->StoragePools : Tenant->StoragePools)) { // N.B. only provide schemeshard with pools that have at least one allocated group if (pr.second->State != TTenantsManager::TStoragePool::NOT_ALLOCATED && pr.second->AllocatedNumGroups > 0) @@ -542,7 +542,7 @@ public: BLOG_D("TSubDomainManip(" << Tenant->Path << ") alter subdomain version " << Version); auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); + request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); request->Record.SetExecTimeoutPeriod(Max<ui64>()); if (Tenant->UserToken.GetUserSID()) @@ -570,7 +570,7 @@ public: BLOG_D("TSubDomainManip(" << Tenant->Path << ") create subdomain"); auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); + request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); request->Record.SetExecTimeoutPeriod(Max<ui64>()); if (Tenant->UserToken.GetUserSID()) request->Record.SetUserToken(Tenant->UserToken.SerializeAsString()); @@ -596,7 +596,7 @@ public: BLOG_D("TSubDomainManip(" << Tenant->Path << ") drop subdomain"); auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); + request->Record.SetDatabaseName(TString(ExtractDomain(Subdomain.first))); request->Record.SetExecTimeoutPeriod(Max<ui64>()); if (Tenant->UserToken.GetUserSID()) request->Record.SetUserToken(Tenant->UserToken.SerializeAsString()); @@ -713,8 +713,8 @@ public: BLOG_D("TSubdomainManip(" << Tenant->Path << ")::Bootstrap"); if (Action == CREATE) { - Become(&TThis::StateSubdomain); - CreateSubdomain(ctx); + Become(&TThis::StateSubdomain); + CreateSubdomain(ctx); } else if (Action == CONFIGURE) { Become(&TThis::StateSubdomain); AlterSubdomain(ctx); @@ -724,7 +724,7 @@ public: } else { Y_VERIFY(Action == REMOVE); Become(&TThis::StateSubdomain); - ReadSubdomainKey(ctx); + ReadSubdomainKey(ctx); } } @@ -809,22 +809,22 @@ public: BLOG_D("TSubdomainManip(" << Tenant->Path << ") got describe result: " << rec.ShortDebugString()); - if (Action == REMOVE) { - switch (rec.GetStatus()) { + if (Action == REMOVE) { + switch (rec.GetStatus()) { case NKikimrScheme::EStatus::StatusPathDoesNotExist: - ActionFinished(ctx); - break; + ActionFinished(ctx); + break; case NKikimrScheme::EStatus::StatusSuccess: - DropSubdomain(ctx); - break; - default: - ReplyAndDie(new TTenantsManager::TEvPrivate::TEvSubdomainFailed(Tenant, rec.GetReason()), ctx); - break; - } - - return; - } - + DropSubdomain(ctx); + break; + default: + ReplyAndDie(new TTenantsManager::TEvPrivate::TEvSubdomainFailed(Tenant, rec.GetReason()), ctx); + break; + } + + return; + } + if (rec.GetStatus() != NKikimrScheme::EStatus::StatusSuccess) { BLOG_ERROR("TSubdomainManip(" << Tenant->Path << ") " << "Receive TEvDescribeSchemeResult with bad status " @@ -1184,7 +1184,7 @@ TTenantsManager::TTenant::TTenant(const TString &path, , IsExternalSubdomain(false) , IsExternalHive(false) , IsExternalSysViewProcessor(false) - , AreResourcesShared(false) + , AreResourcesShared(false) { } @@ -1246,7 +1246,7 @@ bool TTenantsManager::TTenant::HasPoolsToDelete() const bool TTenantsManager::TTenant::HasSubDomainKey() const { - return bool(DomainId); + return bool(DomainId); } TString TTenantsManager::TTenant::MakeStoragePoolName(const TString &poolTypeName) @@ -1317,7 +1317,7 @@ void TTenantsManager::TTenant::RemoveComputationalUnits() void TTenantsManager::ClearState() { Tenants.clear(); - TenantIdToName.clear(); + TenantIdToName.clear(); RemovedTenants.clear(); SlotStats.Clear(); } @@ -1381,22 +1381,22 @@ TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) return nullptr; } -TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) -{ - auto it = TenantIdToName.find(domainId); - if (it != TenantIdToName.end()) - return GetTenant(it->second); - return nullptr; -} - +TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) +{ + auto it = TenantIdToName.find(domainId); + if (it != TenantIdToName.end()) + return GetTenant(it->second); + return nullptr; +} + void TTenantsManager::AddTenant(TTenant::TPtr tenant) { Y_VERIFY(!Tenants.contains(tenant->Path)); Tenants[tenant->Path] = tenant; - if (tenant->DomainId) { - Y_VERIFY(!TenantIdToName.contains(tenant->DomainId)); - TenantIdToName[tenant->DomainId] = tenant->Path; - } + if (tenant->DomainId) { + Y_VERIFY(!TenantIdToName.contains(tenant->DomainId)); + TenantIdToName[tenant->DomainId] = tenant->Path; + } SlotStats.AllocateSlots(tenant->Slots); Counters.Set(COUNTER_TENANTS, Tenants.size()); @@ -1412,15 +1412,15 @@ void TTenantsManager::AddTenant(TTenant::TPtr tenant) void TTenantsManager::RemoveTenant(TTenant::TPtr tenant) { Tenants.erase(tenant->Path); - TenantIdToName.erase(tenant->DomainId); + TenantIdToName.erase(tenant->DomainId); SlotStats.DeallocateSlots(tenant->Slots); - if (tenant->SharedDomainId) { - auto sharedTenant = GetTenant(tenant->SharedDomainId); - Y_VERIFY(sharedTenant); - sharedTenant->HostedTenants.erase(tenant); - } - + if (tenant->SharedDomainId) { + auto sharedTenant = GetTenant(tenant->SharedDomainId); + Y_VERIFY(sharedTenant); + sharedTenant->HostedTenants.erase(tenant); + } + Counters.Set(COUNTER_TENANTS, Tenants.size()); Counters.RemoveUnits(tenant->ComputationalUnits); for (auto &pr : tenant->RegisteredComputationalUnits) @@ -1721,7 +1721,7 @@ TTenantsManager::TStoragePool::TPtr TTenantsManager::MakeStoragePool(TTenant::TP TStoragePool::TPtr pool = new TStoragePool(poolName, kind, size, config); if (tenant->HasSubDomainKey()) - pool->SetScopeId(tenant->DomainId); + pool->SetScopeId(tenant->DomainId); return pool; } @@ -1859,12 +1859,12 @@ void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDataba else status.set_state(Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED); - auto resources = tenant->AreResourcesShared ? - status.mutable_required_shared_resources() : - status.mutable_required_resources(); - + auto resources = tenant->AreResourcesShared ? + status.mutable_required_shared_resources() : + status.mutable_required_resources(); + for (auto &pr : tenant->StoragePools) { - auto &pool = *resources->add_storage_units(); + auto &pool = *resources->add_storage_units(); pool.set_unit_kind(pr.second->Kind); pool.set_count(pr.second->Config.GetNumGroups()); if (pr.second->AllocatedNumGroups) { @@ -1875,18 +1875,18 @@ void TTenantsManager::FillTenantStatus(TTenant::TPtr tenant, Ydb::Cms::GetDataba } for (auto &pr : tenant->ComputationalUnits) { - auto &unit = *resources->add_computational_units(); + auto &unit = *resources->add_computational_units(); unit.set_unit_kind(pr.first.first); unit.set_availability_zone(pr.first.second); unit.set_count(pr.second); } - if (tenant->SharedDomainId) { - auto sharedTenant = GetTenant(tenant->SharedDomainId); - Y_VERIFY(sharedTenant); - status.mutable_serverless_resources()->set_shared_database_path(sharedTenant->Path); - } - + if (tenant->SharedDomainId) { + auto sharedTenant = GetTenant(tenant->SharedDomainId); + Y_VERIFY(sharedTenant); + status.mutable_serverless_resources()->set_shared_database_path(sharedTenant->Path); + } + for (auto &pr : tenant->RegisteredComputationalUnits) { auto &unit = *status.add_registered_resources(); unit.set_host(pr.second.Host); @@ -1956,8 +1956,8 @@ void TTenantsManager::ConfigureTenantSubDomain(TTenant::TPtr tenant, const TActo Y_VERIFY(tenant->IsRunning() || tenant->IsConfiguring()); if (tenant->SubdomainVersion != tenant->ConfirmedSubdomain && !tenant->Worker) { - - auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CONFIGURE, GetTenant(tenant->SharedDomainId)); + + auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CONFIGURE, GetTenant(tenant->SharedDomainId)); tenant->Worker = ctx.RegisterWithSameMailbox(actor); } } @@ -1966,7 +1966,7 @@ void TTenantsManager::CreateTenantSubDomain(TTenant::TPtr tenant, const TActorCo { Y_VERIFY(tenant->State == TTenant::CREATING_SUBDOMAIN); Y_VERIFY(!tenant->Worker); - auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CREATE, GetTenant(tenant->SharedDomainId)); + auto *actor = new TSubDomainManip(SelfId(), tenant, TSubDomainManip::CREATE, GetTenant(tenant->SharedDomainId)); tenant->Worker = ctx.RegisterWithSameMailbox(actor); } @@ -2119,15 +2119,15 @@ void TTenantsManager::SendTenantNotifications(TTenant::TPtr tenant, tenant->Subscribers.clear(); } -TString DomainIdToStringSafe(const TTenantsManager::TDomainId& value) { - // <invalid> cannot be printed inside <pre> tag - if (!value) { - return "[Invalid]"; - } - - return value.ToString(); -} - +TString DomainIdToStringSafe(const TTenantsManager::TDomainId& value) { + // <invalid> cannot be printed inside <pre> tag + if (!value) { + return "[Invalid]"; + } + + return value.ToString(); +} + void TTenantsManager::DumpStateHTML(IOutputStream &os) { HTML(os) { @@ -2163,9 +2163,9 @@ void TTenantsManager::DumpStateHTML(IOutputStream &os) << " Worker: " << tenant->Worker << Endl << " SubdomainVersion: " << tenant->SubdomainVersion << Endl << " ConfirmedSubdomain: " << tenant->ConfirmedSubdomain << Endl - << " DomainId: " << DomainIdToStringSafe(tenant->DomainId) << Endl - << " AreResourcesShared: " << tenant->AreResourcesShared << Endl - << " SharedDomainId: " << DomainIdToStringSafe(tenant->SharedDomainId) << Endl + << " DomainId: " << DomainIdToStringSafe(tenant->DomainId) << Endl + << " AreResourcesShared: " << tenant->AreResourcesShared << Endl + << " SharedDomainId: " << DomainIdToStringSafe(tenant->SharedDomainId) << Endl << " Attributes: " << tenant->Attributes.ShortDebugString() << Endl; os << " Storage pools:" << Endl; @@ -2247,10 +2247,10 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, << " generation=" << tenant->Generation << " errorcode=" << tenant->ErrorCode << " isExternalSubDomain=" << tenant->IsExternalSubdomain - << " isExternalHive=" << tenant->IsExternalHive + << " isExternalHive=" << tenant->IsExternalHive << " isExternalSysViewProcessor=" << tenant->IsExternalSysViewProcessor - << " areResourcesShared=" << tenant->AreResourcesShared - << " sharedDomainId=" << tenant->SharedDomainId); + << " areResourcesShared=" << tenant->AreResourcesShared + << " sharedDomainId=" << tenant->SharedDomainId); NIceDb::TNiceDb db(txc.DB); db.Table<Schema::Tenants>().Key(tenant->Path) @@ -2268,17 +2268,17 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, NIceDb::TUpdate<Schema::Tenants::Generation>(tenant->Generation), NIceDb::TUpdate<Schema::Tenants::ErrorCode>(tenant->ErrorCode), NIceDb::TUpdate<Schema::Tenants::IsExternalSubDomain>(tenant->IsExternalSubdomain), - NIceDb::TUpdate<Schema::Tenants::IsExternalHive>(tenant->IsExternalHive), + NIceDb::TUpdate<Schema::Tenants::IsExternalHive>(tenant->IsExternalHive), NIceDb::TUpdate<Schema::Tenants::IsExternalSysViewProcessor>(tenant->IsExternalSysViewProcessor), NIceDb::TUpdate<Schema::Tenants::AreResourcesShared>(tenant->AreResourcesShared), NIceDb::TUpdate<Schema::Tenants::CreateIdempotencyKey>(tenant->CreateIdempotencyKey)); - if (tenant->SharedDomainId) { - db.Table<Schema::Tenants>().Key(tenant->Path) - .Update(NIceDb::TUpdate<Schema::Tenants::SharedDomainSchemeShardId>(tenant->SharedDomainId.OwnerId), - NIceDb::TUpdate<Schema::Tenants::SharedDomainPathId>(tenant->SharedDomainId.LocalPathId)); - } - + if (tenant->SharedDomainId) { + db.Table<Schema::Tenants>().Key(tenant->Path) + .Update(NIceDb::TUpdate<Schema::Tenants::SharedDomainSchemeShardId>(tenant->SharedDomainId.OwnerId), + NIceDb::TUpdate<Schema::Tenants::SharedDomainPathId>(tenant->SharedDomainId.LocalPathId)); + } + if (tenant->SchemaOperationQuotas) { TString serialized; Y_VERIFY(tenant->SchemaOperationQuotas->SerializeToString(&serialized)); @@ -2323,18 +2323,18 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant, } } -template <typename SchemeShardIdColumn, typename PathIdColumn, typename TRowSet> -static TTenantsManager::TDomainId LoadDomainId(const TRowSet& rowset) { - const ui64 schemeShardId = rowset.template GetValueOrDefault<SchemeShardIdColumn>(0); - const ui64 pathId = rowset.template GetValueOrDefault<PathIdColumn>(0); - - if (!schemeShardId && !pathId) { - return TTenantsManager::TDomainId(); - } - - return TTenantsManager::TDomainId(schemeShardId, pathId); -} - +template <typename SchemeShardIdColumn, typename PathIdColumn, typename TRowSet> +static TTenantsManager::TDomainId LoadDomainId(const TRowSet& rowset) { + const ui64 schemeShardId = rowset.template GetValueOrDefault<SchemeShardIdColumn>(0); + const ui64 pathId = rowset.template GetValueOrDefault<PathIdColumn>(0); + + if (!schemeShardId && !pathId) { + return TTenantsManager::TDomainId(); + } + + return TTenantsManager::TDomainId(schemeShardId, pathId); +} + bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::CMS_TENANTS, "Loading tenants state"); @@ -2366,15 +2366,15 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext ui64 confirmedSubdomain = tenantRowset.GetValueOrDefault<Schema::Tenants::ConfirmedSubdomain>(0); NKikimrSchemeOp::TAlterUserAttributes attrs = tenantRowset.GetValueOrDefault<Schema::Tenants::Attributes>({}); ui64 generation = tenantRowset.GetValueOrDefault<Schema::Tenants::Generation>(1); - const TDomainId domainId = LoadDomainId<Schema::Tenants::SchemeShardId, Schema::Tenants::PathId>(tenantRowset); - const TDomainId sharedDomainId = LoadDomainId<Schema::Tenants::SharedDomainSchemeShardId, Schema::Tenants::SharedDomainPathId>(tenantRowset); + const TDomainId domainId = LoadDomainId<Schema::Tenants::SchemeShardId, Schema::Tenants::PathId>(tenantRowset); + const TDomainId sharedDomainId = LoadDomainId<Schema::Tenants::SharedDomainSchemeShardId, Schema::Tenants::SharedDomainPathId>(tenantRowset); TString issue = tenantRowset.GetValueOrDefault<Schema::Tenants::Issue>(""); Ydb::StatusIds::StatusCode errorCode = static_cast<Ydb::StatusIds::StatusCode>(tenantRowset.GetValueOrDefault<Schema::Tenants::ErrorCode>(0)); bool isExternalSubDomain = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSubDomain>(false); bool isExternalHive = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalHive>(false); bool isExternalSysViewProcessor = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSysViewProcessor>(false); - const bool areResourcesShared = tenantRowset.GetValueOrDefault<Schema::Tenants::AreResourcesShared>(false); + const bool areResourcesShared = tenantRowset.GetValueOrDefault<Schema::Tenants::AreResourcesShared>(false); TTenant::TPtr tenant = new TTenant(path, state, userToken); tenant->Coordinators = coordinators; @@ -2386,14 +2386,14 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext tenant->ConfirmedSubdomain = confirmedSubdomain; tenant->Attributes.CopyFrom(attrs); tenant->Generation = generation; - tenant->DomainId = domainId; - tenant->SharedDomainId = sharedDomainId; + tenant->DomainId = domainId; + tenant->SharedDomainId = sharedDomainId; tenant->ErrorCode = errorCode; tenant->Issue = issue; tenant->IsExternalSubdomain = isExternalSubDomain; tenant->IsExternalHive = isExternalHive; tenant->IsExternalSysViewProcessor = isExternalSysViewProcessor; - tenant->AreResourcesShared = areResourcesShared; + tenant->AreResourcesShared = areResourcesShared; if (tenantRowset.HaveValue<Schema::Tenants::SchemaOperationQuotas>()) { auto& deserialized = tenant->SchemaOperationQuotas.ConstructInPlace(); @@ -2421,16 +2421,16 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext return false; } - for (auto [_, tenant] : Tenants) { - if (!tenant->SharedDomainId) { - continue; - } - - auto sharedTenant = GetTenant(tenant->SharedDomainId); - Y_VERIFY(sharedTenant); - sharedTenant->HostedTenants.emplace(tenant); - } - + for (auto [_, tenant] : Tenants) { + if (!tenant->SharedDomainId) { + continue; + } + + auto sharedTenant = GetTenant(tenant->SharedDomainId); + Y_VERIFY(sharedTenant); + sharedTenant->HostedTenants.emplace(tenant); + } + while (!removedRowset.EndOfSet()) { TRemovedTenant tenant; tenant.Path = removedRowset.GetValue<Schema::RemovedTenants::Path>(); @@ -2834,7 +2834,7 @@ void TTenantsManager::DbUpdateTenantUserToken(TTenant::TPtr tenant, const TActorContext &ctx) { LOG_TRACE_S(ctx, NKikimrServices::CMS_TENANTS, - "Update user token in database for " << tenant->Path); + "Update user token in database for " << tenant->Path); NIceDb::TNiceDb db(txc.DB); db.Table<Schema::Tenants>().Key(tenant->Path) diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index 28210675cad..a21615dd622 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -46,7 +46,7 @@ private: public: using TUnitsCount = THashMap<std::pair<TString, TString>, ui64>; using TSlotsCount = THashMap<TSlotDescription, ui64>; - using TDomainId = TPathId; + using TDomainId = TPathId; ////////////////////////////////////////////////// // Counters @@ -320,11 +320,11 @@ public: SetScopeId(Config, schemeShardId, pathId); } - void SetScopeId(const TDomainId& domainId) - { - SetScopeId(Config, domainId.OwnerId, domainId.LocalPathId); - } - + void SetScopeId(const TDomainId& domainId) + { + SetScopeId(Config, domainId.OwnerId, domainId.LocalPathId); + } + static void SetScopeId(NKikimrBlobStorage::TDefineStoragePool &config, ui64 schemeShardId, ui64 pathId) @@ -514,13 +514,13 @@ public: // Current generation. ui64 Generation; // Subdomain ID. - TDomainId DomainId; - TDomainId SharedDomainId; + TDomainId DomainId; + TDomainId SharedDomainId; bool IsExternalSubdomain; bool IsExternalHive; bool IsExternalSysViewProcessor; - bool AreResourcesShared; - THashSet<TTenant::TPtr> HostedTenants; + bool AreResourcesShared; + THashSet<TTenant::TPtr> HostedTenants; TMaybe<Ydb::Cms::SchemaOperationQuotas> SchemaOperationQuotas; TMaybe<Ydb::Cms::DatabaseQuotas> DatabaseQuotas; @@ -738,7 +738,7 @@ public: TTenant::TPtr FindComputationalUnitKindUsage(const TString &kind, const TString &zone); TTenant::TPtr GetTenant(const TString &name); - TTenant::TPtr GetTenant(const TDomainId &domainId); + TTenant::TPtr GetTenant(const TDomainId &domainId); void AddTenant(TTenant::TPtr tenant); void RemoveTenant(TTenant::TPtr tenant); void RemoveTenantFailed(TTenant::TPtr tenant, @@ -986,7 +986,7 @@ private: TDomainsInfo::TDomain::TPtr Domain; TActorId TenantSlotBrokerPipe; THashMap<TString, TTenant::TPtr> Tenants; - THashMap<TDomainId, TString> TenantIdToName; + THashMap<TDomainId, TString> TenantIdToName; THashMap<TString, TRemovedTenant> RemovedTenants; TTenantsConfig Config; TTxProcessor::TPtr TxProcessor; diff --git a/ydb/core/cms/console/console_ut_tenants.cpp b/ydb/core/cms/console/console_ut_tenants.cpp index 52cd48505d5..57e4820af72 100644 --- a/ydb/core/cms/console/console_ut_tenants.cpp +++ b/ydb/core/cms/console/console_ut_tenants.cpp @@ -654,18 +654,18 @@ Y_UNIT_TEST_SUITE(TConsoleTxProcessorTests) { Y_UNIT_TEST_SUITE(TConsoleTests) { - void RunTestCreateTenant(TTenantTestRuntime& runtime, bool shared = false) { - using EType = TCreateTenantRequest::EType; - - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME, shared ? EType::Shared : EType::Common) - .WithSlots({{SLOT1_TYPE, ZONE1, 3}, {SLOT2_TYPE, ZONE1, 2}, {SLOT3_TYPE, ZONE1, 1}}) - .WithPools({{"hdd", 1}, {"hdd-1", 2}})); - + void RunTestCreateTenant(TTenantTestRuntime& runtime, bool shared = false) { + using EType = TCreateTenantRequest::EType; + + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME, shared ? EType::Shared : EType::Common) + .WithSlots({{SLOT1_TYPE, ZONE1, 3}, {SLOT2_TYPE, ZONE1, 2}, {SLOT3_TYPE, ZONE1, 1}}) + .WithPools({{"hdd", 1}, {"hdd-1", 2}})); + runtime.WaitForHiveState({{{DOMAIN1_NAME, 8, 8, 8}, {TENANT1_1_NAME, 10, 10, 10}}}); - CheckTenantStatus(runtime, TENANT1_1_NAME, shared, Ydb::StatusIds::SUCCESS, + CheckTenantStatus(runtime, TENANT1_1_NAME, shared, Ydb::StatusIds::SUCCESS, Ydb::Cms::GetDatabaseStatusResult::RUNNING, {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, SLOT1_TYPE, ZONE1, 3, 3, @@ -690,56 +690,56 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { RunTestCreateTenant(runtime); } - Y_UNIT_TEST(TestCreateSharedTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - RunTestCreateTenant(runtime, true); - } - - Y_UNIT_TEST(TestCreateServerlessTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // create serverless tenant - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_TENANTS, 2); - CheckCounter(runtime, {}, TTenantsManager::COUNTER_CREATE_REQUESTS, 2); - CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_CREATE_RESPONSES, 2); - } - - Y_UNIT_TEST(TestCreateServerlessTenantWrongSharedDb) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // Empty shared db path - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath("")); - // Unknown shared db - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - } - + Y_UNIT_TEST(TestCreateSharedTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + RunTestCreateTenant(runtime, true); + } + + Y_UNIT_TEST(TestCreateServerlessTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // create serverless tenant + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_TENANTS, 2); + CheckCounter(runtime, {}, TTenantsManager::COUNTER_CREATE_REQUESTS, 2); + CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_CREATE_RESPONSES, 2); + } + + Y_UNIT_TEST(TestCreateServerlessTenantWrongSharedDb) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // Empty shared db path + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath("")); + // Unknown shared db + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + } + void RunTestCreateTenantWrongName(TTenantTestRuntime& runtime) { // Empty path - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("")); // Root path - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/")); // Root path - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("///")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("///")); // Wrong char. - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("tenant?")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("tenant?")); // Wrong domain - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/wrong-domain/tenant")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/wrong-domain/tenant")); // Tenant name starts with non-alpha - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/users/1-tenant")); - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/users/-tenant")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/users/1-tenant")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/users/-tenant")); // Dir name starts with non-alpha - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/1users/tenant")); - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/_users/tenant")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/1users/tenant")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("/dc-1/_users/tenant")); CheckCounter(runtime, {}, TTenantsManager::COUNTER_TENANTS, 0); CheckCounter(runtime, {}, TTenantsManager::COUNTER_CREATE_REQUESTS, 9); @@ -757,9 +757,9 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { } void RunTestCreateTenantWrongPool(TTenantTestRuntime& runtime) { - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME)); - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 0}})); - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"unknown", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME)); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 0}})); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"unknown", 1}})); } Y_UNIT_TEST(TestCreateTenantWrongPool) { @@ -909,8 +909,8 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { } void RunTestAlterTenantModifyComputationalResourcesForPending(TTenantTestRuntime& runtime) { - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}})); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, Ydb::Cms::GetDatabaseStatusResult::PENDING_RESOURCES, {{"hdd", 1, 1}}, {}); @@ -1007,8 +1007,8 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { } void RunTestAlterTenantModifyStorageResourcesForPending(TTenantTestRuntime& runtime) { - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}, {"hdd-1", 3}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}, {"hdd-1", 3}})); CheckCounter(runtime, {{ {"kind", "hdd"} }}, TTenantsManager::COUNTER_REQUESTED_STORAGE_UNITS, 1); CheckCounter(runtime, {{ {"kind", "hdd"} }}, TTenantsManager::COUNTER_ALLOCATED_STORAGE_UNITS, 1); @@ -1159,58 +1159,58 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { RunTestAlterUnknownTenant(runtime); } - Y_UNIT_TEST(TestAlterStorageUnitsOfSharedTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - - // alter shared tenant (extend existent storage pool) - CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - {{"hdd", 1}, {"hdd-1", 1}}, {}); - // check status - CheckTenantStatus(runtime, TENANT1_1_NAME, true, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 2, 2}, {"hdd-1", 3, 3}}, {}, - SLOT1_TYPE, ZONE1, 3, 3, - SLOT2_TYPE, ZONE1, 2, 2, - SLOT3_TYPE, ZONE1, 1, 1); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 1); - CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); - - // alter shared tenant (add new storage pool) - CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::UNSUPPORTED, - {{"hdd-2", 1}}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 2); - CheckCounter(runtime, {{ {"status", "UNSUPPORTED"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); - - // restart and try again - RestartConsole(runtime); - // alter shared tenant - CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::UNSUPPORTED, - {{"hdd-2", 1}}, {}); - } - - Y_UNIT_TEST(TestAlterServerlessTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // create serverless tenant - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); - // alter serverless tenant + Y_UNIT_TEST(TestAlterStorageUnitsOfSharedTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + + // alter shared tenant (extend existent storage pool) + CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + {{"hdd", 1}, {"hdd-1", 1}}, {}); + // check status + CheckTenantStatus(runtime, TENANT1_1_NAME, true, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 2, 2}, {"hdd-1", 3, 3}}, {}, + SLOT1_TYPE, ZONE1, 3, 3, + SLOT2_TYPE, ZONE1, 2, 2, + SLOT3_TYPE, ZONE1, 1, 1); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 1); + CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); + + // alter shared tenant (add new storage pool) + CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::UNSUPPORTED, + {{"hdd-2", 1}}, {}); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 2); + CheckCounter(runtime, {{ {"status", "UNSUPPORTED"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); + + // restart and try again + RestartConsole(runtime); + // alter shared tenant + CheckAlterTenantPools(runtime, TENANT1_1_NAME, Ydb::StatusIds::UNSUPPORTED, + {{"hdd-2", 1}}, {}); + } + + Y_UNIT_TEST(TestAlterServerlessTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // create serverless tenant + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); + // alter serverless tenant CheckAlterTenantSlots(runtime, TENANT1_2_NAME, Ydb::StatusIds::BAD_REQUEST, {{ {SLOT1_TYPE, ZONE_ANY, 1} }}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 1); - CheckCounter(runtime, {{ {"status", "BAD_REQUEST"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); - } - + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_ALTER_REQUESTS, 1); + CheckCounter(runtime, {{ {"status", "BAD_REQUEST"} }}, TTenantsManager::COUNTER_ALTER_RESPONSES, 1); + } + void RunTestListTenants(TTenantTestRuntime& runtime) { CheckCreateTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, @@ -1522,26 +1522,26 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { runtime.WaitForHiveState({{{DOMAIN1_NAME, 8, 8, 8}, {TENANT1_1_NAME, 10, 10, 10}}}); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, - SLOT1_TYPE, ZONE1, 3, 3, - SLOT2_TYPE, ZONE1, 2, 2, - SLOT3_TYPE, ZONE1, 1, 1); - + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, + SLOT1_TYPE, ZONE1, 3, 3, + SLOT2_TYPE, ZONE1, 2, 2, + SLOT3_TYPE, ZONE1, 1, 1); + CheckAlterRegisteredUnits(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{ {"host1", 1, "kind1"}, {"host2", 2, "kind2"} }}, {}); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, - {{"host1", 1, "kind1"}, {"host2", 2, "kind2"}}, - SLOT1_TYPE, ZONE1, 3, 3, - SLOT2_TYPE, ZONE1, 2, 2, - SLOT3_TYPE, ZONE1, 1, 1); - + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, + {{"host1", 1, "kind1"}, {"host2", 2, "kind2"}}, + SLOT1_TYPE, ZONE1, 3, 3, + SLOT2_TYPE, ZONE1, 2, 2, + SLOT3_TYPE, ZONE1, 1, 1); + CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, @@ -1566,13 +1566,13 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { runtime.WaitForHiveState({{{DOMAIN1_NAME, 8, 8, 8}, {TENANT1_1_NAME, 10, 10, 10}}}); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, - SLOT1_TYPE, ZONE1, 3, 3, - SLOT2_TYPE, ZONE1, 2, 2, - SLOT3_TYPE, ZONE1, 1, 1); - + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}, {"hdd-1", 2, 2}}, {}, + SLOT1_TYPE, ZONE1, 3, 3, + SLOT2_TYPE, ZONE1, 2, 2, + SLOT3_TYPE, ZONE1, 1, 1); + CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); // Restart to check we don't load any garbage for removed tenant. @@ -1594,78 +1594,78 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { RunTestRemoveTenant(runtime); } - Y_UNIT_TEST(TestRemoveSharedTenantWoServerlessTenants) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // remove shared tenant - CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); - CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); - } - - Y_UNIT_TEST(TestRemoveSharedTenantWithServerlessTenants) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // create serverless tenant - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); - // remove shared tenant - CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::PRECONDITION_FAILED); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); - CheckCounter(runtime, {{ {"status", "PRECONDITION_FAILED"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); - } - - Y_UNIT_TEST(TestRemoveSharedTenantAfterRemoveServerlessTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // create serverless tenant - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); - // remove serverless tenant - CheckRemoveTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); - // remove shared tenant - CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 2); - CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 2); - } - - Y_UNIT_TEST(TestRemoveServerlessTenant) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); - // create shared tenant - RunTestCreateTenant(runtime, true); - // create serverless tenant - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithSharedDbPath(TENANT1_1_NAME)); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); - // remove serverless tenant - CheckRemoveTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); - // check counters - CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); - CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); - } - + Y_UNIT_TEST(TestRemoveSharedTenantWoServerlessTenants) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // remove shared tenant + CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); + CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); + } + + Y_UNIT_TEST(TestRemoveSharedTenantWithServerlessTenants) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // create serverless tenant + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); + // remove shared tenant + CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::PRECONDITION_FAILED); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); + CheckCounter(runtime, {{ {"status", "PRECONDITION_FAILED"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); + } + + Y_UNIT_TEST(TestRemoveSharedTenantAfterRemoveServerlessTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // create serverless tenant + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); + // remove serverless tenant + CheckRemoveTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + // remove shared tenant + CheckRemoveTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS); + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 2); + CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 2); + } + + Y_UNIT_TEST(TestRemoveServerlessTenant) { + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), {}, true); + // create shared tenant + RunTestCreateTenant(runtime, true); + // create serverless tenant + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithSharedDbPath(TENANT1_1_NAME)); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, {}, {}); + // remove serverless tenant + CheckRemoveTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + // check counters + CheckCounter(runtime, {}, TTenantsManager::COUNTER_REMOVE_REQUESTS, 1); + CheckCounter(runtime, {{ {"status", "SUCCESS"} }}, TTenantsManager::COUNTER_REMOVE_RESPONSES, 1); + } + void RunTestCreateSubSubDomain(TTenantTestRuntime& runtime) { CheckCreateTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}, {"hdd-1", 2}}, @@ -1751,8 +1751,8 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { Y_UNIT_TEST(TestRegisterComputationalUnitsForPending) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}, {"hdd-1", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME).WithPools({{"hdd", 1}, {"hdd-1", 1}})); runtime.WaitForHiveState({{{DOMAIN1_NAME, 8, 8, 8}}}); @@ -1785,8 +1785,8 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {{"hdd", 1, 1}, {"hdd-1", 1, 1}}, {{"host2", 2, "kind2"}, {"host3", 3, "kind2"}}); - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME).WithPools({{"hdd", 1}, {"hdd-1", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME).WithPools({{"hdd", 1}, {"hdd-1", 1}})); CheckAlterRegisteredUnits(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, {{ {"host4", 4, "kind1"}, @@ -1935,11 +1935,11 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { // with no TEvNotifyOperationCompletionResponse. CheckNotificationRequest(runtime, id, Ydb::StatusIds::GENERIC_ERROR); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT1_TYPE, ZONE1, 1, 1); - + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT1_TYPE, ZONE1, 1, 1); + // Send tenant removal command and store operation id. runtime.SetObserverFunc(CatchPoolEvent(captured)); id = SendTenantRemovalCommand(runtime, TENANT1_1_NAME); @@ -2160,10 +2160,10 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { {{"hdd", 1}}, SLOT1_TYPE, ZONE1, 1); // Wrong request shouldn't eat quota. - CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("")); + CheckCreateTenant(runtime, Ydb::StatusIds::BAD_REQUEST, TCreateTenantRequest("")); // This one should fail on subdomain creation and then release quota. - CheckCreateTenant(runtime, Ydb::StatusIds::GENERIC_ERROR, - TCreateTenantRequest(TENANT1_1_NAME + "/sub").WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::GENERIC_ERROR, + TCreateTenantRequest(TENANT1_1_NAME + "/sub").WithPools({{"hdd", 1}})); // Here status is received before tenant is completely removed. // Wait for complete removal. WaitForTenantStatus(runtime, TENANT1_1_NAME + "/sub", Ydb::StatusIds::NOT_FOUND); @@ -2210,10 +2210,10 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckCreateTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, SLOT3_TYPE, ZONE_ANY, 5); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT3_TYPE, ZONE_ANY, 5, 5); + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT3_TYPE, ZONE_ANY, 5, 5); // Wrong request shouldn't eat quota. CheckCreateTenant(runtime, "", Ydb::StatusIds::BAD_REQUEST, {{"hdd", 1}}, @@ -2224,16 +2224,16 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { SLOT2_TYPE, ZONE_ANY, 5); // Wait for complete tenant removal. WaitForTenantStatus(runtime, TENANT1_1_NAME + "/sub", Ydb::StatusIds::NOT_FOUND); - CheckTenantStatus(runtime, TENANT1_1_NAME + "/sub", Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + CheckTenantStatus(runtime, TENANT1_1_NAME + "/sub", Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); // OK. CheckCreateTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, SLOT2_TYPE, ZONE_ANY, 5); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT2_TYPE, ZONE_ANY, 5, 5); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT2_TYPE, ZONE_ANY, 5, 5); // Out of quota. CheckCreateTenant(runtime, TENANT1_3_NAME, Ydb::StatusIds::UNAVAILABLE, {{"hdd", 1}}, @@ -2254,23 +2254,23 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckAlterTenantSlots(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, {{ {SLOT3_TYPE, ZONE_ANY, 1} }}, {}); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT2_TYPE, ZONE_ANY, 5, 5, - SLOT3_TYPE, ZONE_ANY, 1, 1); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT2_TYPE, ZONE_ANY, 5, 5, + SLOT3_TYPE, ZONE_ANY, 1, 1); // Remove tenant to release some resources. CheckRemoveTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); // OK. CheckCreateTenant(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, SLOT2_TYPE, ZONE_ANY, 6); - CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT2_TYPE, ZONE_ANY, 6, 6); + CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT2_TYPE, ZONE_ANY, 6, 6); // Out of quota. CheckAlterTenantSlots(runtime, TENANT1_1_NAME, Ydb::StatusIds::UNAVAILABLE, {{ {SLOT3_TYPE, ZONE_ANY, 1} }}, @@ -2308,10 +2308,10 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckCreateTenant(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, SLOT2_TYPE, ZONE_ANY, 5); - CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT2_TYPE, ZONE_ANY, 5, 5); + CheckTenantStatus(runtime, TENANT1_1_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT2_TYPE, ZONE_ANY, 5, 5); // Out of quota for SLOT2_TYPE. CheckCreateTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::UNAVAILABLE, {{"hdd", 1}}, @@ -2320,10 +2320,10 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckCreateTenant(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, SLOT2_TYPE, ZONE1, 1); - CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - SLOT2_TYPE, ZONE1, 1, 1); + CheckTenantStatus(runtime, TENANT1_2_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + SLOT2_TYPE, ZONE1, 1, 1); // Out of total quota. CheckCreateTenant(runtime, TENANT1_3_NAME, Ydb::StatusIds::UNAVAILABLE, {{"hdd", 1}}, @@ -2332,10 +2332,10 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckCreateTenant(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, {{"hdd", 1}}, "any", ZONE_ANY, 12); - CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - "any", ZONE_ANY, 12, 12); + CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + "any", ZONE_ANY, 12, 12); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE_ANY } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 5); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE1 } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 1); @@ -2357,11 +2357,11 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { CheckAlterTenantSlots(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, {{ {SLOT2_TYPE, ZONE_ANY, 1} }}, {{ {"any", ZONE_ANY, 1} }}); - CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, - Ydb::Cms::GetDatabaseStatusResult::RUNNING, - {{"hdd", 1, 1}}, {}, - "any", ZONE_ANY, 11, 11, - SLOT2_TYPE, ZONE_ANY, 1, 1); + CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS, + Ydb::Cms::GetDatabaseStatusResult::RUNNING, + {{"hdd", 1, 1}}, {}, + "any", ZONE_ANY, 11, 11, + SLOT2_TYPE, ZONE_ANY, 1, 1); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE_ANY } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 5); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE1 } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 1); @@ -2369,8 +2369,8 @@ Y_UNIT_TEST_SUITE(TConsoleTests) { // Remove tenant to release some resources. CheckRemoveTenant(runtime, TENANT1_3_NAME, Ydb::StatusIds::SUCCESS); - CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::NOT_FOUND, - Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); + CheckTenantStatus(runtime, TENANT1_3_NAME, Ydb::StatusIds::NOT_FOUND, + Ydb::Cms::GetDatabaseStatusResult::STATE_UNSPECIFIED, {}, {}); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE_ANY } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 4); CheckCounter(runtime, {{ {"kind", SLOT2_TYPE}, {"zone", ZONE1 } }}, TTenantsManager::COUNTER_COMPUTATIONAL_UNITS, 1); diff --git a/ydb/core/cms/console/log_settings_configurator_ut.cpp b/ydb/core/cms/console/log_settings_configurator_ut.cpp index 844c334ceee..03b52e27171 100644 --- a/ydb/core/cms/console/log_settings_configurator_ut.cpp +++ b/ydb/core/cms/console/log_settings_configurator_ut.cpp @@ -62,12 +62,12 @@ NKikimrConsole::TConfigItem ITEM_TENANT2_TYPE1_LOG_1; TVector<TComponentSettings> InitLogSettingsConfigurator(TTenantTestRuntime &runtime) { - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_1_NAME) - .WithPools({{"hdd", 1}})); - CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, - TCreateTenantRequest(TENANT1_2_NAME) - .WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_1_NAME) + .WithPools({{"hdd", 1}})); + CheckCreateTenant(runtime, Ydb::StatusIds::SUCCESS, + TCreateTenantRequest(TENANT1_2_NAME) + .WithPools({{"hdd", 1}})); ChangeTenant(runtime, TENANT1_1_NAME); diff --git a/ydb/core/cms/console/ut_helpers.h b/ydb/core/cms/console/ut_helpers.h index ff3101752cc..9faf6468fa2 100644 --- a/ydb/core/cms/console/ut_helpers.h +++ b/ydb/core/cms/console/ut_helpers.h @@ -231,11 +231,11 @@ void CheckListConfigSubscriptions(TTenantTestRuntime &runtime, Ydb::StatusIds::S inline bool CompareState(THashMap<std::pair<TString, TString>, TSlotState> slots, THashMap<TString, TPoolAllocation> pools, THashMap<std::pair<TString, ui32>, TUnitRegistration> registrations, - const Ydb::Cms::GetDatabaseStatusResult &status, bool shared = false) + const Ydb::Cms::GetDatabaseStatusResult &status, bool shared = false) { - const auto& resources = shared ? status.required_shared_resources() : status.required_resources(); - - for (auto &unit : resources.computational_units()) { + const auto& resources = shared ? status.required_shared_resources() : status.required_resources(); + + for (auto &unit : resources.computational_units()) { auto key = std::make_pair(unit.unit_kind(), unit.availability_zone()); auto count = unit.count(); if (!slots.contains(key)) @@ -245,7 +245,7 @@ inline bool CompareState(THashMap<std::pair<TString, TString>, TSlotState> slots slots[key].Required = 0; } - for (auto &unit : resources.storage_units()) { + for (auto &unit : resources.storage_units()) { auto key = unit.unit_kind(); auto size = unit.count(); if (!pools.contains(key)) @@ -301,7 +301,7 @@ inline bool CompareState(THashMap<std::pair<TString, TString>, TSlotState> slots } template <typename ...Ts> -void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, bool shared, +void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, bool shared, Ydb::StatusIds::StatusCode code, Ydb::Cms::GetDatabaseStatusResult::State state, TVector<TPoolAllocation> poolTypes, @@ -337,7 +337,7 @@ void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, bool sh UNIT_ASSERT_VALUES_EQUAL(status.path(), CanonizePath(path)); - ok = status.state() == state && CompareState(slots, pools, registrations, status, shared); + ok = status.state() == state && CompareState(slots, pools, registrations, status, shared); if (!ok) { TDispatchOptions options; options.FinalEvents.emplace_back(NConsole::TTenantsManager::TEvPrivate::EvRetryAllocateResources); @@ -353,17 +353,17 @@ void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, bool sh } } -template <typename ...Ts> -void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, - Ydb::StatusIds::StatusCode code, - Ydb::Cms::GetDatabaseStatusResult::State state, - TVector<TPoolAllocation> poolTypes, - TVector<TUnitRegistration> unitRegistrations, - Ts... args) -{ - CheckTenantStatus(runtime, path, false, code, state, poolTypes, unitRegistrations, args...); -} - +template <typename ...Ts> +void CheckTenantStatus(TTenantTestRuntime &runtime, const TString &path, + Ydb::StatusIds::StatusCode code, + Ydb::Cms::GetDatabaseStatusResult::State state, + TVector<TPoolAllocation> poolTypes, + TVector<TUnitRegistration> unitRegistrations, + Ts... args) +{ + CheckTenantStatus(runtime, path, false, code, state, poolTypes, unitRegistrations, args...); +} + inline void WaitForTenantStatus(TTenantTestRuntime &runtime, const TString &path, Ydb::StatusIds::StatusCode code) diff --git a/ydb/core/cms/http.cpp b/ydb/core/cms/http.cpp index fcca57b1b89..648d864b6ed 100644 --- a/ydb/core/cms/http.cpp +++ b/ydb/core/cms/http.cpp @@ -140,37 +140,37 @@ private: ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(response.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); } - static TString DumpRequest(const NMonitoring::IMonHttpRequest& request) { - TStringBuilder result; - result << "{"; - - result << " Method: " << request.GetMethod() - << " Uri: " << request.GetUri(); - - result << " Headers {"; - for (const auto& header : request.GetHeaders()) { + static TString DumpRequest(const NMonitoring::IMonHttpRequest& request) { + TStringBuilder result; + result << "{"; + + result << " Method: " << request.GetMethod() + << " Uri: " << request.GetUri(); + + result << " Headers {"; + for (const auto& header : request.GetHeaders()) { if (stricmp(header.Name().data(), "Authorization") == 0) { - continue; - } - - result << " " << header.ToString(); - } - result << " }"; - - result << " Body: " << request.GetPostContent().Head(1000); - - result << " }"; - return result; - } - + continue; + } + + result << " " << header.ToString(); + } + result << " }"; + + result << " Body: " << request.GetPostContent().Head(1000); + + result << " }"; + return result; + } + void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ctx); NMon::TEvHttpInfo *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::CMS, "HTTP request" - << ": dump# " << DumpRequest(msg->Request)); + LOG_DEBUG_S(ctx, NKikimrServices::CMS, "HTTP request" + << ": dump# " << DumpRequest(msg->Request)); // Check for API call. if (msg->Request.GetPathInfo().StartsWith("/api/")) { diff --git a/ydb/core/cms/info_collector.cpp b/ydb/core/cms/info_collector.cpp index d1d9bc06ae5..02766b3e497 100644 --- a/ydb/core/cms/info_collector.cpp +++ b/ydb/core/cms/info_collector.cpp @@ -1,435 +1,435 @@ #include "cms_impl.h" -#include "info_collector.h" +#include "info_collector.h" #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/blobstorage/base/blobstorage_events.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/interconnect.h> -#include <library/cpp/actors/core/log.h> - -#define LOG_T(stream) LOG_TRACE_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) -#define LOG_D(stream) LOG_DEBUG_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) -#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) -#define LOG_E(stream) LOG_ERROR_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) - +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/interconnect.h> +#include <library/cpp/actors/core/log.h> + +#define LOG_T(stream) LOG_TRACE_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) +#define LOG_D(stream) LOG_DEBUG_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) +#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) +#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) +#define LOG_E(stream) LOG_ERROR_S (*TlsActivationContext, NKikimrServices::CMS, "[InfoCollector] " << stream) + namespace NKikimr { namespace NCms { -using namespace NNodeWhiteboard; -using namespace NKikimrWhiteboard; - -class TInfoCollector: public TActorBootstrapped<TInfoCollector> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CMS_INFO_COLLECTOR; - } - - explicit TInfoCollector(const TActorId& client, const TDuration& timeout) - : Client(client) - , Timeout(timeout) - , Info(new TClusterInfo) - , BaseConfigReceived(false) - { +using namespace NNodeWhiteboard; +using namespace NKikimrWhiteboard; + +class TInfoCollector: public TActorBootstrapped<TInfoCollector> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CMS_INFO_COLLECTOR; + } + + explicit TInfoCollector(const TActorId& client, const TDuration& timeout) + : Client(client) + , Timeout(timeout) + , Info(new TClusterInfo) + , BaseConfigReceived(false) + { } - void Bootstrap(); - -private: - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvents::TEvWakeup, ReplyAndDie); - - // Nodes - hFunc(TEvInterconnect::TEvNodesInfo, Handle); - - // BSC - hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - - // Whiteboard & TenantPool - hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle); - hFunc(TEvWhiteboard::TEvTabletStateResponse, Handle); - hFunc(TEvWhiteboard::TEvPDiskStateResponse, Handle); - hFunc(TEvWhiteboard::TEvVDiskStateResponse, Handle); - hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle); - hFunc(TEvents::TEvUndelivered, Handle); - hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); - IgnoreFunc(TEvInterconnect::TEvNodeConnected); - - default: - LOG_E("Unexpected event" - << ": type# " << ev->GetTypeRewrite() - << ", event# " << (ev->HasEvent() ? ev->GetBase()->ToString() : "serialized?")); - } + void Bootstrap(); + +private: + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + sFunc(TEvents::TEvWakeup, ReplyAndDie); + + // Nodes + hFunc(TEvInterconnect::TEvNodesInfo, Handle); + + // BSC + hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + + // Whiteboard & TenantPool + hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle); + hFunc(TEvWhiteboard::TEvTabletStateResponse, Handle); + hFunc(TEvWhiteboard::TEvPDiskStateResponse, Handle); + hFunc(TEvWhiteboard::TEvVDiskStateResponse, Handle); + hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle); + hFunc(TEvents::TEvUndelivered, Handle); + hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); + IgnoreFunc(TEvInterconnect::TEvNodeConnected); + + default: + LOG_E("Unexpected event" + << ": type# " << ev->GetTypeRewrite() + << ", event# " << (ev->HasEvent() ? ev->GetBase()->ToString() : "serialized?")); + } } - void ReplyAndDie(); - void MaybeReplyAndDie(); - void PassAway() override; - - // Nodes - void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev); - - // BSC - void RequestBaseConfig(); - void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev); - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev); - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev); - void OnPipeDestroyed(); - - // Whiteboard & TenantPool - void SendNodeRequests(ui32 nodeId); - void SendNodeEvent(ui32 nodeId, const TActorId& recipient, IEventBase* request, ui32 responseType); - bool IsNodeInfoRequired(ui32 nodeId, ui32 eventType) const; - void ResponseProcessed(ui32 nodeId, ui32 eventType); - void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev); - void Handle(TEvWhiteboard::TEvTabletStateResponse::TPtr& ev); - void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev); - void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev); - void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev); - void Handle(TEvents::TEvUndelivered::TPtr& ev); - void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev); - -private: - const TActorId Client; - const TDuration Timeout; - - TClusterInfoPtr Info; - TActorId BscPipe; - bool BaseConfigReceived; - THashMap<ui32, TSet<ui32>> NodeEvents; // nodeId -> expected events - THashMap<TPDiskID, TPDiskStateInfo, TPDiskIDHash> PDiskInfo; - THashMap<TVDiskID, TVDiskStateInfo> VDiskInfo; - -}; // TInfoCollector - -void TInfoCollector::ReplyAndDie() { - auto ev = MakeHolder<TCms::TEvPrivate::TEvClusterInfo>(); - ev->Success = BaseConfigReceived; - - if (BaseConfigReceived) { - for (const auto& [id, info] : PDiskInfo) { - Info->UpdatePDiskState(id, info); - } - - for (const auto& [id, info] : VDiskInfo) { - Info->UpdateVDiskState(id, info); - } - - ev->Info = Info; - ev->Info->SetTimestamp(TlsActivationContext->Now()); + void ReplyAndDie(); + void MaybeReplyAndDie(); + void PassAway() override; + + // Nodes + void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev); + + // BSC + void RequestBaseConfig(); + void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev); + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev); + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev); + void OnPipeDestroyed(); + + // Whiteboard & TenantPool + void SendNodeRequests(ui32 nodeId); + void SendNodeEvent(ui32 nodeId, const TActorId& recipient, IEventBase* request, ui32 responseType); + bool IsNodeInfoRequired(ui32 nodeId, ui32 eventType) const; + void ResponseProcessed(ui32 nodeId, ui32 eventType); + void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev); + void Handle(TEvWhiteboard::TEvTabletStateResponse::TPtr& ev); + void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev); + void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev); + void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev); + void Handle(TEvents::TEvUndelivered::TPtr& ev); + void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev); + +private: + const TActorId Client; + const TDuration Timeout; + + TClusterInfoPtr Info; + TActorId BscPipe; + bool BaseConfigReceived; + THashMap<ui32, TSet<ui32>> NodeEvents; // nodeId -> expected events + THashMap<TPDiskID, TPDiskStateInfo, TPDiskIDHash> PDiskInfo; + THashMap<TVDiskID, TVDiskStateInfo> VDiskInfo; + +}; // TInfoCollector + +void TInfoCollector::ReplyAndDie() { + auto ev = MakeHolder<TCms::TEvPrivate::TEvClusterInfo>(); + ev->Success = BaseConfigReceived; + + if (BaseConfigReceived) { + for (const auto& [id, info] : PDiskInfo) { + Info->UpdatePDiskState(id, info); + } + + for (const auto& [id, info] : VDiskInfo) { + Info->UpdateVDiskState(id, info); + } + + ev->Info = Info; + ev->Info->SetTimestamp(TlsActivationContext->Now()); } - Send(Client, std::move(ev)); - PassAway(); + Send(Client, std::move(ev)); + PassAway(); } -void TInfoCollector::MaybeReplyAndDie() { - if (!BaseConfigReceived) { +void TInfoCollector::MaybeReplyAndDie() { + if (!BaseConfigReceived) { return; - } + } - for (const auto& [nodeId, events] : NodeEvents) { - if (!events.empty()) { + for (const auto& [nodeId, events] : NodeEvents) { + if (!events.empty()) { return; - } - } - - ReplyAndDie(); + } + } + + ReplyAndDie(); } -void TInfoCollector::PassAway() { - for (const auto& [nodeId, _] : NodeEvents) { - Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe()); - } - - if (BscPipe) { - NTabletPipe::CloseAndForgetClient(SelfId(), BscPipe); - } - - TActorBootstrapped::PassAway(); +void TInfoCollector::PassAway() { + for (const auto& [nodeId, _] : NodeEvents) { + Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe()); + } + + if (BscPipe) { + NTabletPipe::CloseAndForgetClient(SelfId(), BscPipe); + } + + TActorBootstrapped::PassAway(); } -void TInfoCollector::Bootstrap() { - Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); - Schedule(Timeout, new TEvents::TEvWakeup()); - Become(&TThis::StateWork); -} +void TInfoCollector::Bootstrap() { + Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); + Schedule(Timeout, new TEvents::TEvWakeup()); + Become(&TThis::StateWork); +} -void TInfoCollector::Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { - RequestBaseConfig(); +void TInfoCollector::Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { + RequestBaseConfig(); - for (const auto& node : ev->Get()->Nodes) { - Info->AddNode(node, &TlsActivationContext->AsActorContext()); - SendNodeRequests(node.NodeId); + for (const auto& node : ev->Get()->Nodes) { + Info->AddNode(node, &TlsActivationContext->AsActorContext()); + SendNodeRequests(node.NodeId); } -} - -void TInfoCollector::RequestBaseConfig() { - using namespace NTabletPipe; - - const auto domains = AppData()->DomainsInfo->Domains; - Y_VERIFY(domains.size() <= 1); - - for (const auto& domain : domains) { - const auto bscId = MakeBSControllerID(domain.second->DefaultStateStorageGroup); - BscPipe = Register(CreateClient(SelfId(), bscId, TClientConfig(TClientRetryPolicy::WithRetries()))); - - auto ev = MakeHolder<TEvBlobStorage::TEvControllerConfigRequest>(); - ev->Record.MutableRequest()->AddCommand()->MutableQueryBaseConfig(); - SendData(SelfId(), BscPipe, ev.Release()); - } } -void TInfoCollector::Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - LOG_D("Got base config" - << ": record# " << record.ShortDebugString()); - - if (!record.GetSuccess() || !record.StatusSize() || !record.GetStatus(0).GetSuccess()) { - LOG_E("Couldn't get base config"); - ReplyAndDie(); - } else { - BaseConfigReceived = true; - - for (const auto& pdisk : record.GetStatus(0).GetBaseConfig().GetPDisk()) { - Info->AddPDisk(pdisk); - } - - for (const auto& vdisk : record.GetStatus(0).GetBaseConfig().GetVSlot()) { - Info->AddVDisk(vdisk); - } - - for (const auto& group : record.GetStatus(0).GetBaseConfig().GetGroup()) { - Info->AddBSGroup(group); - } - - MaybeReplyAndDie(); +void TInfoCollector::RequestBaseConfig() { + using namespace NTabletPipe; + + const auto domains = AppData()->DomainsInfo->Domains; + Y_VERIFY(domains.size() <= 1); + + for (const auto& domain : domains) { + const auto bscId = MakeBSControllerID(domain.second->DefaultStateStorageGroup); + BscPipe = Register(CreateClient(SelfId(), bscId, TClientConfig(TClientRetryPolicy::WithRetries()))); + + auto ev = MakeHolder<TEvBlobStorage::TEvControllerConfigRequest>(); + ev->Record.MutableRequest()->AddCommand()->MutableQueryBaseConfig(); + SendData(SelfId(), BscPipe, ev.Release()); } -} +} + +void TInfoCollector::Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + LOG_D("Got base config" + << ": record# " << record.ShortDebugString()); + + if (!record.GetSuccess() || !record.StatusSize() || !record.GetStatus(0).GetSuccess()) { + LOG_E("Couldn't get base config"); + ReplyAndDie(); + } else { + BaseConfigReceived = true; + + for (const auto& pdisk : record.GetStatus(0).GetBaseConfig().GetPDisk()) { + Info->AddPDisk(pdisk); + } + + for (const auto& vdisk : record.GetStatus(0).GetBaseConfig().GetVSlot()) { + Info->AddVDisk(vdisk); + } + + for (const auto& group : record.GetStatus(0).GetBaseConfig().GetGroup()) { + Info->AddBSGroup(group); + } -void TInfoCollector::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - const auto& msg = *ev->Get(); - if (msg.ClientId == BscPipe && msg.Status != NKikimrProto::OK) { - OnPipeDestroyed(); - } + MaybeReplyAndDie(); + } } -void TInfoCollector::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { - const auto& msg = *ev->Get(); - if (msg.ClientId == BscPipe) { - OnPipeDestroyed(); +void TInfoCollector::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + const auto& msg = *ev->Get(); + if (msg.ClientId == BscPipe && msg.Status != NKikimrProto::OK) { + OnPipeDestroyed(); } -} - -void TInfoCollector::OnPipeDestroyed() { - LOG_W("BscPipe destroyed"); - - if (BscPipe) { - NTabletPipe::CloseAndForgetClient(SelfId(), BscPipe); - } - - if (!BaseConfigReceived) { - RequestBaseConfig(); - } } -void TInfoCollector::SendNodeRequests(ui32 nodeId) { - const TActorId whiteBoardId = MakeNodeWhiteboardServiceId(nodeId); - SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvSystemStateRequest(), TEvWhiteboard::EvSystemStateResponse); - SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvTabletStateRequest(), TEvWhiteboard::EvTabletStateResponse); - SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvPDiskStateRequest(), TEvWhiteboard::EvPDiskStateResponse); - SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvVDiskStateRequest(), TEvWhiteboard::EvVDiskStateResponse); +void TInfoCollector::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { + const auto& msg = *ev->Get(); + if (msg.ClientId == BscPipe) { + OnPipeDestroyed(); + } +} - const auto domains = AppData()->DomainsInfo->Domains; - Y_VERIFY(domains.size() <= 1); +void TInfoCollector::OnPipeDestroyed() { + LOG_W("BscPipe destroyed"); - for (const auto& domain : domains) { - const TActorId tenantPoolId = MakeTenantPoolID(nodeId, domain.second->DomainUid); - SendNodeEvent(nodeId, tenantPoolId, new TEvTenantPool::TEvGetStatus(true), TEvTenantPool::EvTenantPoolStatus); + if (BscPipe) { + NTabletPipe::CloseAndForgetClient(SelfId(), BscPipe); } -} -void TInfoCollector::SendNodeEvent(ui32 nodeId, const TActorId& recipient, IEventBase* request, ui32 responseType) { - Send(recipient, request, IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); - NodeEvents[nodeId].insert(responseType); + if (!BaseConfigReceived) { + RequestBaseConfig(); + } } -bool TInfoCollector::IsNodeInfoRequired(ui32 nodeId, ui32 eventType) const { - auto it = NodeEvents.find(nodeId); - if (it == NodeEvents.end()) { - LOG_W("Got info from unknown node" - << ": nodeId# " << nodeId); - return false; +void TInfoCollector::SendNodeRequests(ui32 nodeId) { + const TActorId whiteBoardId = MakeNodeWhiteboardServiceId(nodeId); + SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvSystemStateRequest(), TEvWhiteboard::EvSystemStateResponse); + SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvTabletStateRequest(), TEvWhiteboard::EvTabletStateResponse); + SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvPDiskStateRequest(), TEvWhiteboard::EvPDiskStateResponse); + SendNodeEvent(nodeId, whiteBoardId, new TEvWhiteboard::TEvVDiskStateRequest(), TEvWhiteboard::EvVDiskStateResponse); + + const auto domains = AppData()->DomainsInfo->Domains; + Y_VERIFY(domains.size() <= 1); + + for (const auto& domain : domains) { + const TActorId tenantPoolId = MakeTenantPoolID(nodeId, domain.second->DomainUid); + SendNodeEvent(nodeId, tenantPoolId, new TEvTenantPool::TEvGetStatus(true), TEvTenantPool::EvTenantPoolStatus); } +} - return it->second.contains(eventType); +void TInfoCollector::SendNodeEvent(ui32 nodeId, const TActorId& recipient, IEventBase* request, ui32 responseType) { + Send(recipient, request, IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); + NodeEvents[nodeId].insert(responseType); } -void TInfoCollector::ResponseProcessed(ui32 nodeId, ui32 eventType) { - Y_VERIFY_S(NodeEvents.contains(nodeId), "Unexpected node" - << ": nodeId# " << nodeId); - Y_VERIFY_S(NodeEvents[nodeId].contains(eventType), "Unexpected event" - << ": nodeId# " << nodeId - << ", eventType# " << eventType); - - NodeEvents[nodeId].erase(eventType); - if (NodeEvents[nodeId].empty()) { - MaybeReplyAndDie(); - } -} - -void TInfoCollector::Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - const auto& record = ev->Get()->Record; - - LOG_D("Got system state" - << ": nodeId# " << nodeId - << ", record# " << record.DebugString()); - - if (!IsNodeInfoRequired(nodeId, ev->Type)) { +bool TInfoCollector::IsNodeInfoRequired(ui32 nodeId, ui32 eventType) const { + auto it = NodeEvents.find(nodeId); + if (it == NodeEvents.end()) { + LOG_W("Got info from unknown node" + << ": nodeId# " << nodeId); + return false; + } + + return it->second.contains(eventType); +} + +void TInfoCollector::ResponseProcessed(ui32 nodeId, ui32 eventType) { + Y_VERIFY_S(NodeEvents.contains(nodeId), "Unexpected node" + << ": nodeId# " << nodeId); + Y_VERIFY_S(NodeEvents[nodeId].contains(eventType), "Unexpected event" + << ": nodeId# " << nodeId + << ", eventType# " << eventType); + + NodeEvents[nodeId].erase(eventType); + if (NodeEvents[nodeId].empty()) { + MaybeReplyAndDie(); + } +} + +void TInfoCollector::Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + const auto& record = ev->Get()->Record; + + LOG_D("Got system state" + << ": nodeId# " << nodeId + << ", record# " << record.DebugString()); + + if (!IsNodeInfoRequired(nodeId, ev->Type)) { return; - } + } - if (record.SystemStateInfoSize() != 1) { - LOG_E("Unexpected system state's size" - << ": nodeId# " << nodeId - << ", size# " << record.SystemStateInfoSize()); + if (record.SystemStateInfoSize() != 1) { + LOG_E("Unexpected system state's size" + << ": nodeId# " << nodeId + << ", size# " << record.SystemStateInfoSize()); return; } - Info->SetNodeState(nodeId, NKikimrCms::UP, record.GetSystemStateInfo(0)); - ResponseProcessed(nodeId, ev->Type); + Info->SetNodeState(nodeId, NKikimrCms::UP, record.GetSystemStateInfo(0)); + ResponseProcessed(nodeId, ev->Type); } -void TInfoCollector::Handle(TEvWhiteboard::TEvTabletStateResponse::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - const auto& record = ev->Get()->Record; +void TInfoCollector::Handle(TEvWhiteboard::TEvTabletStateResponse::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + const auto& record = ev->Get()->Record; - LOG_D("Got tablet state" - << ": nodeId# " << nodeId - << ", record# " << record.DebugString()); + LOG_D("Got tablet state" + << ": nodeId# " << nodeId + << ", record# " << record.DebugString()); - if (!IsNodeInfoRequired(nodeId, ev->Type)) { + if (!IsNodeInfoRequired(nodeId, ev->Type)) { return; - } + } - for (const auto& info : record.GetTabletStateInfo()) { + for (const auto& info : record.GetTabletStateInfo()) { Info->AddTablet(nodeId, info); - } + } - ResponseProcessed(nodeId, ev->Type); + ResponseProcessed(nodeId, ev->Type); } -void TInfoCollector::Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - auto& record = ev->Get()->Record; +void TInfoCollector::Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + auto& record = ev->Get()->Record; - LOG_D("Got PDisk state" - << ": nodeId# " << nodeId - << ", record# " << record.DebugString()); + LOG_D("Got PDisk state" + << ": nodeId# " << nodeId + << ", record# " << record.DebugString()); - if (!IsNodeInfoRequired(nodeId, ev->Type)) { + if (!IsNodeInfoRequired(nodeId, ev->Type)) { return; - } + } - for (ui32 i = 0; i < record.PDiskStateInfoSize(); ++i) { - auto* info = record.MutablePDiskStateInfo(i); - const auto id = TPDiskID(nodeId, info->GetPDiskId()); + for (ui32 i = 0; i < record.PDiskStateInfoSize(); ++i) { + auto* info = record.MutablePDiskStateInfo(i); + const auto id = TPDiskID(nodeId, info->GetPDiskId()); PDiskInfo[id].Swap(info); } - ResponseProcessed(nodeId, ev->Type); + ResponseProcessed(nodeId, ev->Type); } -void TInfoCollector::Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - auto& record = ev->Get()->Record; +void TInfoCollector::Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + auto& record = ev->Get()->Record; - LOG_D("Got VDisk state" - << ": nodeId# " << nodeId - << ", record# " << record.DebugString()); + LOG_D("Got VDisk state" + << ": nodeId# " << nodeId + << ", record# " << record.DebugString()); - if (!IsNodeInfoRequired(nodeId, ev->Type)) { + if (!IsNodeInfoRequired(nodeId, ev->Type)) { return; - } + } - for (ui32 i = 0; i < record.VDiskStateInfoSize(); ++i) { - auto* info = record.MutableVDiskStateInfo(i); - const auto id = VDiskIDFromVDiskID(info->GetVDiskId()); + for (ui32 i = 0; i < record.VDiskStateInfoSize(); ++i) { + auto* info = record.MutableVDiskStateInfo(i); + const auto id = VDiskIDFromVDiskID(info->GetVDiskId()); VDiskInfo[id].Swap(info); } - ResponseProcessed(nodeId, ev->Type); + ResponseProcessed(nodeId, ev->Type); } -void TInfoCollector::Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - const auto& record = ev->Get()->Record; - - LOG_D("Got TenantPoolStatus" - << ": nodeId# " << nodeId - << ", record# " << record.DebugString()); - - if (!IsNodeInfoRequired(nodeId, ev->Type)) { - return; - } - - Info->AddNodeTenants(nodeId, record); - ResponseProcessed(nodeId, ev->Type); +void TInfoCollector::Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + const auto& record = ev->Get()->Record; + + LOG_D("Got TenantPoolStatus" + << ": nodeId# " << nodeId + << ", record# " << record.DebugString()); + + if (!IsNodeInfoRequired(nodeId, ev->Type)) { + return; + } + + Info->AddNodeTenants(nodeId, record); + ResponseProcessed(nodeId, ev->Type); } -void TInfoCollector::Handle(TEvents::TEvUndelivered::TPtr& ev) { - const auto& msg = *ev->Get(); - const ui32 nodeId = ev->Cookie; - - LOG_D("Undelivered" - << ": nodeId# " << nodeId - << ", source# " << msg.SourceType - << ", reason# " << msg.Reason); - - if (!NodeEvents.contains(nodeId)) { - LOG_E("Undelivered to unknown node" - << ": nodeId# " << nodeId); - return; - } - - if (msg.SourceType == TEvTenantPool::EvGetStatus && msg.Reason == TEvents::TEvUndelivered::ReasonActorUnknown) { - LOG_W("Node is alive, but TenantPool is not running (KIKIMR-8249)"); - ResponseProcessed(nodeId, TEvTenantPool::EvTenantPoolStatus); - } else { - Info->ClearNode(nodeId); - NodeEvents[nodeId].clear(); - } - - MaybeReplyAndDie(); +void TInfoCollector::Handle(TEvents::TEvUndelivered::TPtr& ev) { + const auto& msg = *ev->Get(); + const ui32 nodeId = ev->Cookie; + + LOG_D("Undelivered" + << ": nodeId# " << nodeId + << ", source# " << msg.SourceType + << ", reason# " << msg.Reason); + + if (!NodeEvents.contains(nodeId)) { + LOG_E("Undelivered to unknown node" + << ": nodeId# " << nodeId); + return; + } + + if (msg.SourceType == TEvTenantPool::EvGetStatus && msg.Reason == TEvents::TEvUndelivered::ReasonActorUnknown) { + LOG_W("Node is alive, but TenantPool is not running (KIKIMR-8249)"); + ResponseProcessed(nodeId, TEvTenantPool::EvTenantPoolStatus); + } else { + Info->ClearNode(nodeId); + NodeEvents[nodeId].clear(); + } + + MaybeReplyAndDie(); } -void TInfoCollector::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - const ui32 nodeId = ev->Get()->NodeId; +void TInfoCollector::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + const ui32 nodeId = ev->Get()->NodeId; - LOG_D("Disconnected" - << ": nodeId# " << nodeId); + LOG_D("Disconnected" + << ": nodeId# " << nodeId); - if (!NodeEvents.contains(nodeId)) { - LOG_E("Disconnected unknown node" - << ": nodeId# " << nodeId); + if (!NodeEvents.contains(nodeId)) { + LOG_E("Disconnected unknown node" + << ": nodeId# " << nodeId); return; - } + } - Info->ClearNode(nodeId); - NodeEvents[nodeId].clear(); - MaybeReplyAndDie(); + Info->ClearNode(nodeId); + NodeEvents[nodeId].clear(); + MaybeReplyAndDie(); } -IActor* CreateInfoCollector(const TActorId& client, const TDuration& timeout) { +IActor* CreateInfoCollector(const TActorId& client, const TDuration& timeout) { return new TInfoCollector(client, timeout); } diff --git a/ydb/core/cms/info_collector.h b/ydb/core/cms/info_collector.h index 70d14b88e76..ec030909e6b 100644 --- a/ydb/core/cms/info_collector.h +++ b/ydb/core/cms/info_collector.h @@ -5,7 +5,7 @@ namespace NKikimr { namespace NCms { -IActor* CreateInfoCollector(const TActorId& client, const TDuration& timeout); +IActor* CreateInfoCollector(const TActorId& client, const TDuration& timeout); } // NCms } // NKikimr diff --git a/ydb/core/cms/json_proxy.h b/ydb/core/cms/json_proxy.h index 6c752e3f0a5..185d0ce01b6 100644 --- a/ydb/core/cms/json_proxy.h +++ b/ydb/core/cms/json_proxy.h @@ -1,7 +1,7 @@ #pragma once -#include "audit_log.h" - +#include "audit_log.h" + #include <ydb/core/base/defs.h> #include <library/cpp/actors/core/actor.h> @@ -98,7 +98,7 @@ protected: void ReplyWithErrorAndDie(const TString &err, const TActorContext &ctx) { - ReplyAndDieImpl(err, ctx); + ReplyAndDieImpl(err, ctx); } void ReplyAndDie(const typename TResponseEvent::ProtoRecordType &resp, const TActorContext &ctx) @@ -114,13 +114,13 @@ protected: void ReplyAndDie(const TString &json, const TActorContext &ctx) { - ReplyAndDieImpl(TString(NMonitoring::HTTPOKJSON) + json, ctx); - } + ReplyAndDieImpl(TString(NMonitoring::HTTPOKJSON) + json, ctx); + } - void ReplyAndDieImpl(const TString &data, const TActorContext &ctx) - { - AuditLog("JsonProxy", RequestEvent, data, ctx); - ctx.Send(RequestEvent->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + void ReplyAndDieImpl(const TString &data, const TActorContext &ctx) + { + AuditLog("JsonProxy", RequestEvent, data, ctx); + ctx.Send(RequestEvent->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); TBase::Die(ctx); } diff --git a/ydb/core/cms/json_proxy_proto.h b/ydb/core/cms/json_proxy_proto.h index bed01a07dd6..b1b87f30cda 100644 --- a/ydb/core/cms/json_proxy_proto.h +++ b/ydb/core/cms/json_proxy_proto.h @@ -1,7 +1,7 @@ #pragma once -#include "pdisk_state.h" - +#include "pdisk_state.h" + #include <ydb/core/base/defs.h> #include <ydb/core/mon/mon.h> @@ -14,7 +14,7 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_writer.h> -#include <util/generic/serialized_enum.h> +#include <util/generic/serialized_enum.h> #include <iostream> namespace NKikimr { @@ -56,12 +56,12 @@ protected: else if (name == "NKikimrConsole::TConfigItem::EMergeStrategy") return ReplyWithEnumDescription(*NKikimrConsole::TConfigItem::EMergeStrategy_descriptor(), ctx); else if (name == "NKikimrServices::EServiceKikimr") - return ReplyWithLogComponents(ctx.LoggerSettings(), ctx); + return ReplyWithLogComponents(ctx.LoggerSettings(), ctx); else if (name == "NKikimrCms::TCmsConfig::TLogConfig::ELevel") return ReplyWithEnumDescription(*NKikimrCms::TCmsConfig::TLogConfig::ELevel_descriptor(), ctx); else if (name == "NKikimrCms::TLogRecordData::EType") return ReplyWithEnumDescription(*NKikimrCms::TLogRecordData::EType_descriptor(), ctx); - else if (name == "NCms::EPDiskState") + else if (name == "NCms::EPDiskState") return ReplyWithEnumDescription(*NKikimrBlobStorage::TPDiskState::E_descriptor(), ctx); } else if (cgi.Has("type")) { TString name = cgi.Get("type"); @@ -81,27 +81,27 @@ protected: NMon::IEvHttpInfoRes::EContentType::Custom)); } - void ReplyWithLogComponents(NLog::TSettings *settings, const TActorContext &ctx) - { - NJson::TJsonValue json; - - if (settings) { - for (NLog::EComponent i = settings->MinVal; i < settings->MaxVal; i++) { - auto name = settings->ComponentName(i); - if (!*name) { - continue; - } - - NJson::TJsonValue value; - value["name"] = name; - value["number"] = static_cast<ui32>(i); - json["value"].AppendValue(value); - } - } - - Reply(WriteJson(json), ctx); - } - + void ReplyWithLogComponents(NLog::TSettings *settings, const TActorContext &ctx) + { + NJson::TJsonValue json; + + if (settings) { + for (NLog::EComponent i = settings->MinVal; i < settings->MaxVal; i++) { + auto name = settings->ComponentName(i); + if (!*name) { + continue; + } + + NJson::TJsonValue value; + value["name"] = name; + value["number"] = static_cast<ui32>(i); + json["value"].AppendValue(value); + } + } + + Reply(WriteJson(json), ctx); + } + void ReplyWithEnumDescription(const ::google::protobuf::EnumDescriptor &descriptor, const TActorContext &ctx) { diff --git a/ydb/core/cms/log_formatter.h b/ydb/core/cms/log_formatter.h index 5f5f6528885..2ebb9e74702 100644 --- a/ydb/core/cms/log_formatter.h +++ b/ydb/core/cms/log_formatter.h @@ -101,14 +101,14 @@ template<> template<> TString TLogFormatter<NKikimrCms::TEXT_FORMAT_SHORT>::FormatData(const NKikimrCms::TLogRecordData::TPDiskMonitorAction &data) { TStringStream ss; - ss << "Send command to BS_CONTROLLER to change status of PDisk " + ss << "Send command to BS_CONTROLLER to change status of PDisk " << data.GetPDiskId().GetNodeId() << ":" << data.GetPDiskId().GetDiskId() - << " at " << (data.HasHost() ? data.GetHost() : "<undefined>") + << " at " << (data.HasHost() ? data.GetHost() : "<undefined>") << " from " << data.GetCurrentStatus() << " to " << data.GetRequiredStatus(); - if (data.HasReason()) { - ss << "(" << data.GetReason() << ")"; - } + if (data.HasReason()) { + ss << "(" << data.GetReason() << ")"; + } return ss.Str(); } diff --git a/ydb/core/cms/pdisk_state.h b/ydb/core/cms/pdisk_state.h index 7d9111e446c..6dd673a7299 100644 --- a/ydb/core/cms/pdisk_state.h +++ b/ydb/core/cms/pdisk_state.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/node_whiteboard.pb.h> - -namespace NKikimr { -namespace NCms { - -using TPDiskStateInfo = NKikimrWhiteboard::TPDiskStateInfo; + +namespace NKikimr { +namespace NCms { + +using TPDiskStateInfo = NKikimrWhiteboard::TPDiskStateInfo; using EPDiskState = NKikimrBlobStorage::TPDiskState::E; - -} // NCms -} // NKikimr + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/pdiskid.h b/ydb/core/cms/pdiskid.h index 39298cb3bc9..a1ff7b215f9 100644 --- a/ydb/core/cms/pdiskid.h +++ b/ydb/core/cms/pdiskid.h @@ -62,8 +62,8 @@ struct TPDiskIDHash { } // NCms } // NKikimr - -template<> -inline void Out<NKikimr::NCms::TPDiskID>(IOutputStream& o, const NKikimr::NCms::TPDiskID& x) { - o << x.ToString(); -} + +template<> +inline void Out<NKikimr::NCms::TPDiskID>(IOutputStream& o, const NKikimr::NCms::TPDiskID& x) { + o << x.ToString(); +} diff --git a/ydb/core/cms/sentinel.cpp b/ydb/core/cms/sentinel.cpp index a9031bc085f..638b78fe09b 100644 --- a/ydb/core/cms/sentinel.cpp +++ b/ydb/core/cms/sentinel.cpp @@ -1,77 +1,77 @@ -#include "cms_impl.h" -#include "sentinel.h" -#include "sentinel_impl.h" - +#include "cms_impl.h" +#include "sentinel.h" +#include "sentinel_impl.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/blobstorage/base/blobstorage_events.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> #include <ydb/core/protos/services.pb.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash_set.h> -#include <util/generic/map.h> -#include <util/string/builder.h> -#include <util/string/join.h> - -namespace NKikimr { -namespace NCms { - -#if defined LOG_T || \ - defined LOG_D || \ - defined LOG_I || \ - defined LOG_N || \ - defined LOG_W || \ - defined LOG_E || \ - defined LOG_C -#error log macro redefinition -#endif - -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) - -namespace NSentinel { - -/// TPDiskStatusComputer - -TPDiskStatusComputer::TPDiskStatusComputer(const ui32& defaultStateLimit, const TLimitsMap& stateLimits) - : DefaultStateLimit(defaultStateLimit) - , StateLimits(stateLimits) - , StateCounter(0) -{ -} - -void TPDiskStatusComputer::AddState(EPDiskState state) { - if (StateCounter && state == State) { - if (StateCounter != Max<ui64>()) { - ++StateCounter; - } - } else { + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> +#include <util/generic/map.h> +#include <util/string/builder.h> +#include <util/string/join.h> + +namespace NKikimr { +namespace NCms { + +#if defined LOG_T || \ + defined LOG_D || \ + defined LOG_I || \ + defined LOG_N || \ + defined LOG_W || \ + defined LOG_E || \ + defined LOG_C +#error log macro redefinition +#endif + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) +#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CMS, "[Sentinel] [" << Name() << "] " << stream) + +namespace NSentinel { + +/// TPDiskStatusComputer + +TPDiskStatusComputer::TPDiskStatusComputer(const ui32& defaultStateLimit, const TLimitsMap& stateLimits) + : DefaultStateLimit(defaultStateLimit) + , StateLimits(stateLimits) + , StateCounter(0) +{ +} + +void TPDiskStatusComputer::AddState(EPDiskState state) { + if (StateCounter && state == State) { + if (StateCounter != Max<ui64>()) { + ++StateCounter; + } + } else { PrevState = std::exchange(State, state); - StateCounter = 1; - } -} - -EPDiskStatus TPDiskStatusComputer::Compute(EPDiskStatus current, TString& reason) const { - if (!StateCounter) { - reason = "Uninitialized StateCounter"; - return current; - } - - auto it = StateLimits.find(State); - const ui32 stateLimit = (it != StateLimits.end()) ? it->second : DefaultStateLimit; - - if (!stateLimit || StateCounter < stateLimit) { + StateCounter = 1; + } +} + +EPDiskStatus TPDiskStatusComputer::Compute(EPDiskStatus current, TString& reason) const { + if (!StateCounter) { + reason = "Uninitialized StateCounter"; + return current; + } + + auto it = StateLimits.find(State); + const ui32 stateLimit = (it != StateLimits.end()) ? it->second : DefaultStateLimit; + + if (!stateLimit || StateCounter < stateLimit) { reason = TStringBuilder() << " PrevState# " << PrevState << " State# " << State @@ -83,63 +83,63 @@ EPDiskStatus TPDiskStatusComputer::Compute(EPDiskStatus current, TString& reason default: return EPDiskStatus::INACTIVE; } - } - - reason = TStringBuilder() + } + + reason = TStringBuilder() << " PrevState# " << PrevState - << " State# " << State - << " StateCounter# " << StateCounter - << " StateLimit# " << stateLimit; - + << " State# " << State + << " StateCounter# " << StateCounter + << " StateLimit# " << stateLimit; + PrevState = State; - switch (State) { + switch (State) { case NKikimrBlobStorage::TPDiskState::Normal: - return EPDiskStatus::ACTIVE; - default: - return EPDiskStatus::FAULTY; - } -} - -void TPDiskStatusComputer::Reset() { - StateCounter = 0; -} - -/// TPDiskStatus - -TPDiskStatus::TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits) - : TPDiskStatusComputer(defaultStateLimit, stateLimits) - , Current(initialStatus) - , ChangingAllowed(true) -{ -} - -void TPDiskStatus::AddState(EPDiskState state) { - TPDiskStatusComputer::AddState(state); -} - -bool TPDiskStatus::IsChanged(TString& reason) const { - return Current != Compute(Current, reason); -} - -bool TPDiskStatus::IsChanged() const { - TString unused; - return IsChanged(unused); -} - -void TPDiskStatus::ApplyChanges(TString& reason) { - Current = Compute(Current, reason); -} - -void TPDiskStatus::ApplyChanges() { - TString unused; - ApplyChanges(unused); -} - -EPDiskStatus TPDiskStatus::GetStatus() const { - return Current; -} - + return EPDiskStatus::ACTIVE; + default: + return EPDiskStatus::FAULTY; + } +} + +void TPDiskStatusComputer::Reset() { + StateCounter = 0; +} + +/// TPDiskStatus + +TPDiskStatus::TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits) + : TPDiskStatusComputer(defaultStateLimit, stateLimits) + , Current(initialStatus) + , ChangingAllowed(true) +{ +} + +void TPDiskStatus::AddState(EPDiskState state) { + TPDiskStatusComputer::AddState(state); +} + +bool TPDiskStatus::IsChanged(TString& reason) const { + return Current != Compute(Current, reason); +} + +bool TPDiskStatus::IsChanged() const { + TString unused; + return IsChanged(unused); +} + +void TPDiskStatus::ApplyChanges(TString& reason) { + Current = Compute(Current, reason); +} + +void TPDiskStatus::ApplyChanges() { + TString unused; + ApplyChanges(unused); +} + +EPDiskStatus TPDiskStatus::GetStatus() const { + return Current; +} + bool TPDiskStatus::IsNewStatusGood() const { TString unused; switch (Compute(Current, unused)) { @@ -158,51 +158,51 @@ bool TPDiskStatus::IsNewStatusGood() const { } } -bool TPDiskStatus::IsChangingAllowed() const { - return ChangingAllowed; -} - -void TPDiskStatus::AllowChanging() { - if (!ChangingAllowed) { - Reset(); - } - - ChangingAllowed = true; -} - -void TPDiskStatus::DisallowChanging() { - ChangingAllowed = false; -} - -/// TPDiskInfo - -TPDiskInfo::TPDiskInfo(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits) - : TPDiskStatus(initialStatus, defaultStateLimit, stateLimits) -{ - Touch(); -} - -void TPDiskInfo::AddState(EPDiskState state) { - TPDiskStatus::AddState(state); - Touch(); -} - +bool TPDiskStatus::IsChangingAllowed() const { + return ChangingAllowed; +} + +void TPDiskStatus::AllowChanging() { + if (!ChangingAllowed) { + Reset(); + } + + ChangingAllowed = true; +} + +void TPDiskStatus::DisallowChanging() { + ChangingAllowed = false; +} + +/// TPDiskInfo + +TPDiskInfo::TPDiskInfo(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits) + : TPDiskStatus(initialStatus, defaultStateLimit, stateLimits) +{ + Touch(); +} + +void TPDiskInfo::AddState(EPDiskState state) { + TPDiskStatus::AddState(state); + Touch(); +} + /// TClusterMap - + TClusterMap::TClusterMap(TCmsStatePtr state) - : State(state) + : State(state) {} - + void TClusterMap::AddPDisk(const TPDiskID& id) { - Y_VERIFY(State->ClusterInfo->HasNode(id.NodeId)); - Y_VERIFY(State->ClusterInfo->HasPDisk(id)); - const auto& location = State->ClusterInfo->Node(id.NodeId).Location; - + Y_VERIFY(State->ClusterInfo->HasNode(id.NodeId)); + Y_VERIFY(State->ClusterInfo->HasPDisk(id)); + const auto& location = State->ClusterInfo->Node(id.NodeId).Location; + ByDataCenter[location.HasKey(TNodeLocation::TKeys::DataCenter) ? location.GetDataCenterId() : ""].insert(id); ByRoom[location.HasKey(TNodeLocation::TKeys::Module) ? location.GetModuleId() : ""].insert(id); ByRack[location.HasKey(TNodeLocation::TKeys::Rack) ? location.GetRackId() : ""].insert(id); -} - +} + /// TGuardian TGuardian::TGuardian(TCmsStatePtr state, ui32 dataCenterRatio, ui32 roomRatio, ui32 rackRatio) @@ -215,222 +215,222 @@ TGuardian::TGuardian(TCmsStatePtr state, ui32 dataCenterRatio, ui32 roomRatio, u TClusterMap::TPDiskIDSet TGuardian::GetAllowedPDisks(const TClusterMap& all, TString& issues, TPDiskIDSet& disallowed) const { - TPDiskIDSet result; - TStringBuilder issuesBuilder; - - #define LOG_IGNORED(ratio) \ - issuesBuilder \ - << "Ignore state updates due to " << #ratio << "Ratio" \ - << ": changed# " << kv.second.size() \ - << ", total# " << all.By##ratio.at(kv.first).size() \ - << ", ratio# " << ratio##Ratio \ - << ", affected pdisks# " << JoinSeq(", ", kv.second) << Endl - - for (const auto& kv : ByDataCenter) { - Y_VERIFY(all.ByDataCenter.contains(kv.first)); - - if (!kv.first || CheckRatio(kv, all.ByDataCenter, DataCenterRatio)) { + TPDiskIDSet result; + TStringBuilder issuesBuilder; + + #define LOG_IGNORED(ratio) \ + issuesBuilder \ + << "Ignore state updates due to " << #ratio << "Ratio" \ + << ": changed# " << kv.second.size() \ + << ", total# " << all.By##ratio.at(kv.first).size() \ + << ", ratio# " << ratio##Ratio \ + << ", affected pdisks# " << JoinSeq(", ", kv.second) << Endl + + for (const auto& kv : ByDataCenter) { + Y_VERIFY(all.ByDataCenter.contains(kv.first)); + + if (!kv.first || CheckRatio(kv, all.ByDataCenter, DataCenterRatio)) { result.insert(kv.second.begin(), kv.second.end()); - } else { - LOG_IGNORED(DataCenter); + } else { + LOG_IGNORED(DataCenter); disallowed.insert(kv.second.begin(), kv.second.end()); - } - } - - for (const auto& kv : ByRoom) { - Y_VERIFY(all.ByRoom.contains(kv.first)); - - if (kv.first && !CheckRatio(kv, all.ByRoom, RoomRatio)) { - LOG_IGNORED(Room); + } + } + + for (const auto& kv : ByRoom) { + Y_VERIFY(all.ByRoom.contains(kv.first)); + + if (kv.first && !CheckRatio(kv, all.ByRoom, RoomRatio)) { + LOG_IGNORED(Room); disallowed.insert(kv.second.begin(), kv.second.end()); - EraseNodesIf(result, [&room = kv.second](const TPDiskID& id) { - return room.contains(id); - }); - } - } - - for (const auto& kv : ByRack) { - Y_VERIFY(all.ByRack.contains(kv.first)); + EraseNodesIf(result, [&room = kv.second](const TPDiskID& id) { + return room.contains(id); + }); + } + } + + for (const auto& kv : ByRack) { + Y_VERIFY(all.ByRack.contains(kv.first)); // ignore check if there is only one node in a rack if (kv.second.size() == 1) { continue; } - if (kv.first && !CheckRatio(kv, all.ByRack, RackRatio)) { - LOG_IGNORED(Rack); + if (kv.first && !CheckRatio(kv, all.ByRack, RackRatio)) { + LOG_IGNORED(Rack); disallowed.insert(kv.second.begin(), kv.second.end()); - EraseNodesIf(result, [&rack = kv.second](const TPDiskID& id) { - return rack.contains(id); - }); - } - } - - #undef LOG_IGNORED - - issues = issuesBuilder; - return result; -} - -/// Main state -struct TSentinelState: public TSimpleRefCount<TSentinelState> { - using TPtr = TIntrusivePtr<TSentinelState>; - - TMap<TPDiskID, TPDiskInfo> PDisks; -}; - -/// Actors - -template <typename TDerived> -class TSentinelChildBase: public TActorBootstrapped<TDerived> { -protected: - void ConnectBSC() { - auto domains = AppData()->DomainsInfo; - const ui32 domainUid = domains->GetDomainUidByTabletId(CmsState->CmsTabletId); - const ui64 bscId = MakeBSControllerID(domains->GetDefaultStateStorageGroup(domainUid)); - - NTabletPipe::TClientConfig config; + EraseNodesIf(result, [&rack = kv.second](const TPDiskID& id) { + return rack.contains(id); + }); + } + } + + #undef LOG_IGNORED + + issues = issuesBuilder; + return result; +} + +/// Main state +struct TSentinelState: public TSimpleRefCount<TSentinelState> { + using TPtr = TIntrusivePtr<TSentinelState>; + + TMap<TPDiskID, TPDiskInfo> PDisks; +}; + +/// Actors + +template <typename TDerived> +class TSentinelChildBase: public TActorBootstrapped<TDerived> { +protected: + void ConnectBSC() { + auto domains = AppData()->DomainsInfo; + const ui32 domainUid = domains->GetDomainUidByTabletId(CmsState->CmsTabletId); + const ui64 bscId = MakeBSControllerID(domains->GetDefaultStateStorageGroup(domainUid)); + + NTabletPipe::TClientConfig config; config.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries(); - CmsState->BSControllerPipe = this->Register(NTabletPipe::CreateClient(CmsState->CmsActorId, bscId, config)); - } - -public: - using TBase = TSentinelChildBase<TDerived>; - - explicit TSentinelChildBase(const TActorId& parent, TCmsStatePtr cmsState) - : Parent(parent) - , CmsState(cmsState) - , Config(CmsState->Config.SentinelConfig) - { - } - -protected: + CmsState->BSControllerPipe = this->Register(NTabletPipe::CreateClient(CmsState->CmsActorId, bscId, config)); + } + +public: + using TBase = TSentinelChildBase<TDerived>; + + explicit TSentinelChildBase(const TActorId& parent, TCmsStatePtr cmsState) + : Parent(parent) + , CmsState(cmsState) + , Config(CmsState->Config.SentinelConfig) + { + } + +protected: const TActorId Parent; - TCmsStatePtr CmsState; - const TCmsSentinelConfig& Config; - -}; // TSentinelChildBase - -template <typename TEvUpdated, typename TDerived> -class TUpdaterBase: public TSentinelChildBase<TDerived> { -public: - using TBase = TUpdaterBase<TEvUpdated, TDerived>; - -protected: - void Reply() { - this->Send(this->Parent, new TEvUpdated()); - this->PassAway(); - } - -public: - explicit TUpdaterBase(const TActorId& parent, TCmsStatePtr cmsState, TSentinelState::TPtr sentinelState) - : TSentinelChildBase<TDerived>(parent, cmsState) - , SentinelState(sentinelState) - { - for (auto& pdisk : SentinelState->PDisks) { - pdisk.second.ClearTouched(); - } - } - -protected: - TSentinelState::TPtr SentinelState; - -}; // TUpdaterBase - -class TConfigUpdater: public TUpdaterBase<TEvSentinel::TEvConfigUpdated, TConfigUpdater> { - void Retry() { - ++Attempt; - Schedule(Config.RetryUpdateConfig, new TEvSentinel::TEvRetry()); - } - - void RequestBSConfig() { - LOG_D("Request blobstorage config" - << ": attempt# " << Attempt); - - if (!CmsState->BSControllerPipe) { - ConnectBSC(); - } - + TCmsStatePtr CmsState; + const TCmsSentinelConfig& Config; + +}; // TSentinelChildBase + +template <typename TEvUpdated, typename TDerived> +class TUpdaterBase: public TSentinelChildBase<TDerived> { +public: + using TBase = TUpdaterBase<TEvUpdated, TDerived>; + +protected: + void Reply() { + this->Send(this->Parent, new TEvUpdated()); + this->PassAway(); + } + +public: + explicit TUpdaterBase(const TActorId& parent, TCmsStatePtr cmsState, TSentinelState::TPtr sentinelState) + : TSentinelChildBase<TDerived>(parent, cmsState) + , SentinelState(sentinelState) + { + for (auto& pdisk : SentinelState->PDisks) { + pdisk.second.ClearTouched(); + } + } + +protected: + TSentinelState::TPtr SentinelState; + +}; // TUpdaterBase + +class TConfigUpdater: public TUpdaterBase<TEvSentinel::TEvConfigUpdated, TConfigUpdater> { + void Retry() { + ++Attempt; + Schedule(Config.RetryUpdateConfig, new TEvSentinel::TEvRetry()); + } + + void RequestBSConfig() { + LOG_D("Request blobstorage config" + << ": attempt# " << Attempt); + + if (!CmsState->BSControllerPipe) { + ConnectBSC(); + } + auto request = MakeHolder<TEvBlobStorage::TEvControllerConfigRequest>(); - request->Record.MutableRequest()->AddCommand()->MutableQueryBaseConfig(); - NTabletPipe::SendData(SelfId(), CmsState->BSControllerPipe, request.Release()); - } - - void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { - const auto& response = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvBlobStorage::TEvControllerConfigResponse" - << ": response# " << response.ShortDebugString()); - - if (!response.GetSuccess() || !response.StatusSize() || !response.GetStatus(0).GetSuccess()) { - TString error = "<no description>"; - if (response.StatusSize()) { - error = response.GetStatus(0).GetErrorDescription(); - } - - LOG_E("Unsuccesful response from BSC" - << ", size# " << response.StatusSize() - << ", error# " << error); - Retry(); - } else { - auto& pdisks = SentinelState->PDisks; - - for (const auto& pdisk : response.GetStatus(0).GetBaseConfig().GetPDisk()) { - TPDiskID id(pdisk.GetNodeId(), pdisk.GetPDiskId()); - - if (pdisks.contains(id)) { - pdisks.at(id).Touch(); - continue; - } - - pdisks.emplace(id, TPDiskInfo(pdisk.GetDriveStatus(), Config.DefaultStateLimit, Config.StateLimits)); - } - - Reply(); - } - } - - void OnPipeDisconnected() { - LOG_E("Pipe to BSC disconnected"); - Retry(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CMS_SENTINEL_CONFIG_UPDATER_ACTOR; - } - - static TStringBuf Name() { + request->Record.MutableRequest()->AddCommand()->MutableQueryBaseConfig(); + NTabletPipe::SendData(SelfId(), CmsState->BSControllerPipe, request.Release()); + } + + void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { + const auto& response = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvBlobStorage::TEvControllerConfigResponse" + << ": response# " << response.ShortDebugString()); + + if (!response.GetSuccess() || !response.StatusSize() || !response.GetStatus(0).GetSuccess()) { + TString error = "<no description>"; + if (response.StatusSize()) { + error = response.GetStatus(0).GetErrorDescription(); + } + + LOG_E("Unsuccesful response from BSC" + << ", size# " << response.StatusSize() + << ", error# " << error); + Retry(); + } else { + auto& pdisks = SentinelState->PDisks; + + for (const auto& pdisk : response.GetStatus(0).GetBaseConfig().GetPDisk()) { + TPDiskID id(pdisk.GetNodeId(), pdisk.GetPDiskId()); + + if (pdisks.contains(id)) { + pdisks.at(id).Touch(); + continue; + } + + pdisks.emplace(id, TPDiskInfo(pdisk.GetDriveStatus(), Config.DefaultStateLimit, Config.StateLimits)); + } + + Reply(); + } + } + + void OnPipeDisconnected() { + LOG_E("Pipe to BSC disconnected"); + Retry(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CMS_SENTINEL_CONFIG_UPDATER_ACTOR; + } + + static TStringBuf Name() { return "ConfigUpdater"sv; - } - - using TBase::TBase; - - void Bootstrap() { - RequestBSConfig(); - Become(&TThis::StateWork); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - cFunc(TEvSentinel::TEvRetry::EventType, RequestBSConfig); - cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); - - hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - ui32 Attempt = 0; - -}; // TConfigUpdater - -class TStateUpdater: public TUpdaterBase<TEvSentinel::TEvStateUpdated, TStateUpdater> { - using TEvWhiteboard = NNodeWhiteboard::TEvWhiteboard; - - static EPDiskState SafePDiskState(EPDiskState state) { - switch (state) { + } + + using TBase::TBase; + + void Bootstrap() { + RequestBSConfig(); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + cFunc(TEvSentinel::TEvRetry::EventType, RequestBSConfig); + cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); + + hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + ui32 Attempt = 0; + +}; // TConfigUpdater + +class TStateUpdater: public TUpdaterBase<TEvSentinel::TEvStateUpdated, TStateUpdater> { + using TEvWhiteboard = NNodeWhiteboard::TEvWhiteboard; + + static EPDiskState SafePDiskState(EPDiskState state) { + switch (state) { case NKikimrBlobStorage::TPDiskState::Initial: case NKikimrBlobStorage::TPDiskState::InitialFormatRead: case NKikimrBlobStorage::TPDiskState::InitialFormatReadError: @@ -445,629 +445,629 @@ class TStateUpdater: public TUpdaterBase<TEvSentinel::TEvStateUpdated, TStateUpd case NKikimrBlobStorage::TPDiskState::OpenFileError: case NKikimrBlobStorage::TPDiskState::ChunkQuotaError: case NKikimrBlobStorage::TPDiskState::DeviceIoError: - return state; - default: - LOG_C("Unknown pdisk state: " << (ui32)state); + return state; + default: + LOG_C("Unknown pdisk state: " << (ui32)state); return NKikimrBlobStorage::TPDiskState::Unknown; - } - } - - bool AcceptNodeReply(ui32 nodeId) { - auto it = WaitNodes.find(nodeId); - if (it == WaitNodes.end()) { - return false; - } - - WaitNodes.erase(it); - return true; - } - - void MaybeReply() { - if (WaitNodes) { - return; - } - - Reply(); - } - - void MarkNodePDisks(ui32 nodeId, EPDiskState state, bool skipTouched = false) { - auto it = SentinelState->PDisks.lower_bound(TPDiskID(nodeId, 0)); - while (it != SentinelState->PDisks.end() && it->first.NodeId == nodeId) { - if (skipTouched && it->second.IsTouched()) { - ++it; - continue; - } - - Y_VERIFY(!it->second.IsTouched()); - it->second.AddState(state); - ++it; - } - } - - void RequestPDiskState(ui32 nodeId) { + } + } + + bool AcceptNodeReply(ui32 nodeId) { + auto it = WaitNodes.find(nodeId); + if (it == WaitNodes.end()) { + return false; + } + + WaitNodes.erase(it); + return true; + } + + void MaybeReply() { + if (WaitNodes) { + return; + } + + Reply(); + } + + void MarkNodePDisks(ui32 nodeId, EPDiskState state, bool skipTouched = false) { + auto it = SentinelState->PDisks.lower_bound(TPDiskID(nodeId, 0)); + while (it != SentinelState->PDisks.end() && it->first.NodeId == nodeId) { + if (skipTouched && it->second.IsTouched()) { + ++it; + continue; + } + + Y_VERIFY(!it->second.IsTouched()); + it->second.AddState(state); + ++it; + } + } + + void RequestPDiskState(ui32 nodeId) { const TActorId wbId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId); - const ui32 flags = IEventHandle::FlagTrackDelivery; - - LOG_D("Request pdisks state" - << ": nodeId# " << nodeId - << ", wbId# " << wbId); - Send(wbId, new TEvWhiteboard::TEvPDiskStateRequest(), flags, nodeId); - } - - void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { - const ui32 nodeId = ev->Sender.NodeId(); - const auto& record = ev->Get()->Record; - - LOG_D("Handle TEvWhiteboard::TEvPDiskStateResponse" - << ": nodeId# " << nodeId - << ", response# " << record.ShortDebugString()); - - if (!AcceptNodeReply(nodeId)) { - LOG_W("PDisk info from unknown node" - << ": nodeId# " << nodeId); - return; - } - - if (!record.PDiskStateInfoSize()) { - LOG_E("There is no pdisk info" - << ": nodeId# " << nodeId); + const ui32 flags = IEventHandle::FlagTrackDelivery; + + LOG_D("Request pdisks state" + << ": nodeId# " << nodeId + << ", wbId# " << wbId); + Send(wbId, new TEvWhiteboard::TEvPDiskStateRequest(), flags, nodeId); + } + + void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { + const ui32 nodeId = ev->Sender.NodeId(); + const auto& record = ev->Get()->Record; + + LOG_D("Handle TEvWhiteboard::TEvPDiskStateResponse" + << ": nodeId# " << nodeId + << ", response# " << record.ShortDebugString()); + + if (!AcceptNodeReply(nodeId)) { + LOG_W("PDisk info from unknown node" + << ": nodeId# " << nodeId); + return; + } + + if (!record.PDiskStateInfoSize()) { + LOG_E("There is no pdisk info" + << ": nodeId# " << nodeId); MarkNodePDisks(nodeId, NKikimrBlobStorage::TPDiskState::Missing); - } else { - for (const auto& info : record.GetPDiskStateInfo()) { - auto it = SentinelState->PDisks.find(TPDiskID(nodeId, info.GetPDiskId())); - if (it == SentinelState->PDisks.end()) { - continue; - } - - const auto safeState = SafePDiskState(info.GetState()); - LOG_T("SafePDiskState" - << ": pdiskId# " << it->first - << ", original# " << (ui32)info.GetState() - << ", safeState# " << safeState); - - it->second.AddState(safeState); - } - + } else { + for (const auto& info : record.GetPDiskStateInfo()) { + auto it = SentinelState->PDisks.find(TPDiskID(nodeId, info.GetPDiskId())); + if (it == SentinelState->PDisks.end()) { + continue; + } + + const auto safeState = SafePDiskState(info.GetState()); + LOG_T("SafePDiskState" + << ": pdiskId# " << it->first + << ", original# " << (ui32)info.GetState() + << ", safeState# " << safeState); + + it->second.AddState(safeState); + } + MarkNodePDisks(nodeId, NKikimrBlobStorage::TPDiskState::Missing, true); - } - - MaybeReply(); - } - - void Handle(TEvents::TEvUndelivered::TPtr& ev) { - using EReason = TEvents::TEvUndelivered::EReason; - - const ui32 nodeId = ev->Cookie; - const EReason reason = ev->Get()->Reason; - - LOG_D("Handle TEvents::TEvUndelivered" - << ": nodeId# " << nodeId - << ", sourceType# " << ev->Get()->SourceType - << ", reason# " << reason); - - if (!AcceptNodeReply(nodeId)) { - LOG_W("Undelivered to unknown node" - << ": nodeId# " << nodeId); - return; - } - - LOG_E("Cannot get pdisks state" - << ": nodeId# " << nodeId - << ", reason# " << reason); - - switch (reason) { - case EReason::Disconnected: + } + + MaybeReply(); + } + + void Handle(TEvents::TEvUndelivered::TPtr& ev) { + using EReason = TEvents::TEvUndelivered::EReason; + + const ui32 nodeId = ev->Cookie; + const EReason reason = ev->Get()->Reason; + + LOG_D("Handle TEvents::TEvUndelivered" + << ": nodeId# " << nodeId + << ", sourceType# " << ev->Get()->SourceType + << ", reason# " << reason); + + if (!AcceptNodeReply(nodeId)) { + LOG_W("Undelivered to unknown node" + << ": nodeId# " << nodeId); + return; + } + + LOG_E("Cannot get pdisks state" + << ": nodeId# " << nodeId + << ", reason# " << reason); + + switch (reason) { + case EReason::Disconnected: MarkNodePDisks(nodeId, NKikimrBlobStorage::TPDiskState::NodeDisconnected); - break; - - default: + break; + + default: MarkNodePDisks(nodeId, NKikimrBlobStorage::TPDiskState::Unknown); - break; - } - - MaybeReply(); - } - - void TimedOut() { - LOG_E("Timed out" - << ": timeout# " << Config.UpdateStateTimeout); - - while (WaitNodes) { - const ui32 nodeId = *WaitNodes.begin(); - + break; + } + + MaybeReply(); + } + + void TimedOut() { + LOG_E("Timed out" + << ": timeout# " << Config.UpdateStateTimeout); + + while (WaitNodes) { + const ui32 nodeId = *WaitNodes.begin(); + MarkNodePDisks(nodeId, NKikimrBlobStorage::TPDiskState::Timeout); - AcceptNodeReply(nodeId); - } - - MaybeReply(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CMS_SENTINEL_STATE_UPDATER_ACTOR; - } - - static TStringBuf Name() { + AcceptNodeReply(nodeId); + } + + MaybeReply(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CMS_SENTINEL_STATE_UPDATER_ACTOR; + } + + static TStringBuf Name() { return "StateUpdater"sv; - } - - using TBase::TBase; - - void Bootstrap() { - for (const auto& pdisk : SentinelState->PDisks) { - if (WaitNodes.insert(pdisk.first.NodeId).second) { - RequestPDiskState(pdisk.first.NodeId); - } - } - - Become(&TThis::StateWork, Config.UpdateStateTimeout, new TEvSentinel::TEvTimeout()); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - cFunc(TEvSentinel::TEvTimeout::EventType, TimedOut); - - hFunc(TEvWhiteboard::TEvPDiskStateResponse, Handle); - - hFunc(TEvents::TEvUndelivered, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - THashSet<ui32> WaitNodes; - -}; // TStateUpdater - -class TStatusChanger: public TSentinelChildBase<TStatusChanger> { - void Reply(bool success = true) { - Send(Parent, new TEvSentinel::TEvStatusChanged(Id, success)); - PassAway(); - } - - void MaybeRetry() { - if (Attempt++ < Config.ChangeStatusRetries) { - Schedule(Config.RetryChangeStatus, new TEvSentinel::TEvRetry()); - } else { - Reply(false); - } - } - - void RequestStatusChange() { - LOG_D("Change pdisk status" - << ": pdiskId# " << Id - << ", status# " << Status - << ", attempt# " << Attempt); - - if (!CmsState->BSControllerPipe) { - ConnectBSC(); - } - + } + + using TBase::TBase; + + void Bootstrap() { + for (const auto& pdisk : SentinelState->PDisks) { + if (WaitNodes.insert(pdisk.first.NodeId).second) { + RequestPDiskState(pdisk.first.NodeId); + } + } + + Become(&TThis::StateWork, Config.UpdateStateTimeout, new TEvSentinel::TEvTimeout()); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + cFunc(TEvSentinel::TEvTimeout::EventType, TimedOut); + + hFunc(TEvWhiteboard::TEvPDiskStateResponse, Handle); + + hFunc(TEvents::TEvUndelivered, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + THashSet<ui32> WaitNodes; + +}; // TStateUpdater + +class TStatusChanger: public TSentinelChildBase<TStatusChanger> { + void Reply(bool success = true) { + Send(Parent, new TEvSentinel::TEvStatusChanged(Id, success)); + PassAway(); + } + + void MaybeRetry() { + if (Attempt++ < Config.ChangeStatusRetries) { + Schedule(Config.RetryChangeStatus, new TEvSentinel::TEvRetry()); + } else { + Reply(false); + } + } + + void RequestStatusChange() { + LOG_D("Change pdisk status" + << ": pdiskId# " << Id + << ", status# " << Status + << ", attempt# " << Attempt); + + if (!CmsState->BSControllerPipe) { + ConnectBSC(); + } + auto request = MakeHolder<TEvBlobStorage::TEvControllerConfigRequest>(); - auto& command = *request->Record.MutableRequest()->AddCommand()->MutableUpdateDriveStatus(); - command.MutableHostKey()->SetNodeId(Id.NodeId); - command.SetPDiskId(Id.DiskId); - command.SetStatus(Status); - NTabletPipe::SendData(SelfId(), CmsState->BSControllerPipe, request.Release()); - } - - void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { - const auto& response = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvBlobStorage::TEvControllerConfigResponse" - << ": response# " << response.ShortDebugString()); - - if (!response.GetSuccess() || !response.StatusSize() || !response.GetStatus(0).GetSuccess()) { - TString error = "<no description>"; - if (response.StatusSize()) { - error = response.GetStatus(0).GetErrorDescription(); - } - - LOG_E("Unsuccesful response from BSC" - << ", size# " << response.StatusSize() - << ", error# " << error); - MaybeRetry(); - } else { - Reply(); - } - } - - void OnPipeDisconnected() { - LOG_E("Pipe to BSC disconnected"); - MaybeRetry(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CMS_SENTINEL_STATUS_CHANGER_ACTOR; - } - - static TStringBuf Name() { + auto& command = *request->Record.MutableRequest()->AddCommand()->MutableUpdateDriveStatus(); + command.MutableHostKey()->SetNodeId(Id.NodeId); + command.SetPDiskId(Id.DiskId); + command.SetStatus(Status); + NTabletPipe::SendData(SelfId(), CmsState->BSControllerPipe, request.Release()); + } + + void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr& ev) { + const auto& response = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvBlobStorage::TEvControllerConfigResponse" + << ": response# " << response.ShortDebugString()); + + if (!response.GetSuccess() || !response.StatusSize() || !response.GetStatus(0).GetSuccess()) { + TString error = "<no description>"; + if (response.StatusSize()) { + error = response.GetStatus(0).GetErrorDescription(); + } + + LOG_E("Unsuccesful response from BSC" + << ", size# " << response.StatusSize() + << ", error# " << error); + MaybeRetry(); + } else { + Reply(); + } + } + + void OnPipeDisconnected() { + LOG_E("Pipe to BSC disconnected"); + MaybeRetry(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CMS_SENTINEL_STATUS_CHANGER_ACTOR; + } + + static TStringBuf Name() { return "StatusChanger"sv; - } - - explicit TStatusChanger( + } + + explicit TStatusChanger( const TActorId& parent, - TCmsStatePtr state, - const TPDiskID& id, - NKikimrBlobStorage::EDriveStatus status) - : TBase(parent, state) - , Id(id) - , Status(status) - { - } - - void Bootstrap() { - RequestStatusChange(); - Become(&TThis::StateWork); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - cFunc(TEvSentinel::TEvRetry::EventType, RequestStatusChange); - cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); - - hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - const TPDiskID Id; - const NKikimrBlobStorage::EDriveStatus Status; - - ui32 Attempt = 0; - -}; // TStatusChanger - -class TSentinel: public TActorBootstrapped<TSentinel> { - struct TCounters { - using TDynamicCounters = NMonitoring::TDynamicCounters; - using TDynamicCounterPtr = NMonitoring::TDynamicCounterPtr; - - TDynamicCounterPtr Group; - TDynamicCounters::TCounterPtr PDisksTotal; - TDynamicCounters::TCounterPtr PDisksPendingChange; - TDynamicCounters::TCounterPtr PDisksChanged; - TDynamicCounters::TCounterPtr PDisksNotChanged; - - explicit TCounters(TDynamicCounterPtr counters) - : Group(counters) - , PDisksTotal(counters->GetCounter("PDisksTotal", false)) - , PDisksPendingChange(counters->GetCounter("PDisksPendingChange", true)) - , PDisksChanged(counters->GetCounter("PDisksChanged", true)) - , PDisksNotChanged(counters->GetCounter("PDisksNotChanged", true)) - { - } - - ~TCounters() { - Group->ResetCounters(); - } - }; - - struct TUpdaterInfo { + TCmsStatePtr state, + const TPDiskID& id, + NKikimrBlobStorage::EDriveStatus status) + : TBase(parent, state) + , Id(id) + , Status(status) + { + } + + void Bootstrap() { + RequestStatusChange(); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + cFunc(TEvSentinel::TEvRetry::EventType, RequestStatusChange); + cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); + + hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const TPDiskID Id; + const NKikimrBlobStorage::EDriveStatus Status; + + ui32 Attempt = 0; + +}; // TStatusChanger + +class TSentinel: public TActorBootstrapped<TSentinel> { + struct TCounters { + using TDynamicCounters = NMonitoring::TDynamicCounters; + using TDynamicCounterPtr = NMonitoring::TDynamicCounterPtr; + + TDynamicCounterPtr Group; + TDynamicCounters::TCounterPtr PDisksTotal; + TDynamicCounters::TCounterPtr PDisksPendingChange; + TDynamicCounters::TCounterPtr PDisksChanged; + TDynamicCounters::TCounterPtr PDisksNotChanged; + + explicit TCounters(TDynamicCounterPtr counters) + : Group(counters) + , PDisksTotal(counters->GetCounter("PDisksTotal", false)) + , PDisksPendingChange(counters->GetCounter("PDisksPendingChange", true)) + , PDisksChanged(counters->GetCounter("PDisksChanged", true)) + , PDisksNotChanged(counters->GetCounter("PDisksNotChanged", true)) + { + } + + ~TCounters() { + Group->ResetCounters(); + } + }; + + struct TUpdaterInfo { TActorId Id; - TInstant StartedAt; - bool Delayed; - - TUpdaterInfo() { - Clear(); - } - + TInstant StartedAt; + bool Delayed; + + TUpdaterInfo() { + Clear(); + } + void Start(const TActorId& id, const TInstant& now) { - Id = id; - StartedAt = now; - Delayed = false; - } - - void Clear() { + Id = id; + StartedAt = now; + Delayed = false; + } + + void Clear() { Id = TActorId(); - StartedAt = TInstant::Zero(); - Delayed = false; - } - }; - - static TInstant Now() { - return TlsActivationContext->Now(); - } - - template <typename TUpdater> - void StartUpdater(TUpdaterInfo& updater) { - if (ConfigUpdater.Id || StateUpdater.Id) { - LOG_I(TUpdater::Name() << " was delayed"); - updater.Delayed = true; - return; - } - - LOG_D("Start " << TUpdater::Name()); - updater.Start(RegisterWithSameMailbox(new TUpdater(SelfId(), CmsState, SentinelState)), Now()); - } - - template <typename TEvThis, typename TUpdaterThat> - void ScheduleUpdate(TUpdaterInfo& updaterThis, const TDuration& intervalThis, TUpdaterInfo& updaterThat) { - TDuration elapsed = Now() - updaterThis.StartedAt; - TDuration delay; - - if (elapsed > intervalThis) { - delay = TDuration::MilliSeconds(1); - } else { - delay = intervalThis - elapsed; - } - - updaterThis.Clear(); - Schedule(delay, new TEvThis()); - - if (updaterThat.Delayed) { - StartUpdater<TUpdaterThat>(updaterThat); - } - } - - void RemoveUntouched() { - EraseNodesIf(SentinelState->PDisks, [](const auto& kv) { - return !kv.second.IsTouched(); - }); - } - - void EnsureAllTouched() const { - Y_VERIFY(AllOf(SentinelState->PDisks, [](const auto& kv) { - return kv.second.IsTouched(); - })); - } - - void LogStatusChange(const TPDiskID& id, EPDiskStatus status, EPDiskStatus requiredStatus, const TString& reason) { - auto ev = MakeHolder<TCms::TEvPrivate::TEvLogAndSend>(); - - ev->LogData.SetRecordType(NKikimrCms::TLogRecordData::PDISK_MONITOR_ACTION); - auto& action = *ev->LogData.MutablePDiskMonitorAction(); - id.Serialize(action.MutablePDiskId()); - action.SetCurrentStatus(status); - action.SetRequiredStatus(requiredStatus); - - Y_VERIFY(CmsState->ClusterInfo->HasNode(id.NodeId)); - action.SetHost(CmsState->ClusterInfo->Node(id.NodeId).Host); - - if (reason) { - action.SetReason(reason); - } - - Send(CmsState->CmsActorId, std::move(ev)); - } - - void UpdateConfig() { - LOG_D("UpdateConfig"); - StartUpdater<TConfigUpdater>(ConfigUpdater); - } - - void OnConfigUpdated() { - LOG_D("Config was updated in " << (Now() - ConfigUpdater.StartedAt)); - - RemoveUntouched(); - *Counters->PDisksTotal = SentinelState->PDisks.size(); - - ScheduleUpdate<TEvSentinel::TEvUpdateConfig, TStateUpdater>( - ConfigUpdater, Config.UpdateConfigInterval, StateUpdater - ); - } - - void UpdateState() { - LOG_D("UpdateState"); - StartUpdater<TStateUpdater>(StateUpdater); - } - - void OnStateUpdated() { - LOG_D("State was updated in " << (Now() - StateUpdater.StartedAt)); - - EnsureAllTouched(); - - if (!CmsState->ClusterInfo) { - LOG_C("Missing cluster info"); - ScheduleUpdate<TEvSentinel::TEvUpdateState, TConfigUpdater>( - StateUpdater, Config.UpdateStateInterval, ConfigUpdater - ); - - return; - } - + StartedAt = TInstant::Zero(); + Delayed = false; + } + }; + + static TInstant Now() { + return TlsActivationContext->Now(); + } + + template <typename TUpdater> + void StartUpdater(TUpdaterInfo& updater) { + if (ConfigUpdater.Id || StateUpdater.Id) { + LOG_I(TUpdater::Name() << " was delayed"); + updater.Delayed = true; + return; + } + + LOG_D("Start " << TUpdater::Name()); + updater.Start(RegisterWithSameMailbox(new TUpdater(SelfId(), CmsState, SentinelState)), Now()); + } + + template <typename TEvThis, typename TUpdaterThat> + void ScheduleUpdate(TUpdaterInfo& updaterThis, const TDuration& intervalThis, TUpdaterInfo& updaterThat) { + TDuration elapsed = Now() - updaterThis.StartedAt; + TDuration delay; + + if (elapsed > intervalThis) { + delay = TDuration::MilliSeconds(1); + } else { + delay = intervalThis - elapsed; + } + + updaterThis.Clear(); + Schedule(delay, new TEvThis()); + + if (updaterThat.Delayed) { + StartUpdater<TUpdaterThat>(updaterThat); + } + } + + void RemoveUntouched() { + EraseNodesIf(SentinelState->PDisks, [](const auto& kv) { + return !kv.second.IsTouched(); + }); + } + + void EnsureAllTouched() const { + Y_VERIFY(AllOf(SentinelState->PDisks, [](const auto& kv) { + return kv.second.IsTouched(); + })); + } + + void LogStatusChange(const TPDiskID& id, EPDiskStatus status, EPDiskStatus requiredStatus, const TString& reason) { + auto ev = MakeHolder<TCms::TEvPrivate::TEvLogAndSend>(); + + ev->LogData.SetRecordType(NKikimrCms::TLogRecordData::PDISK_MONITOR_ACTION); + auto& action = *ev->LogData.MutablePDiskMonitorAction(); + id.Serialize(action.MutablePDiskId()); + action.SetCurrentStatus(status); + action.SetRequiredStatus(requiredStatus); + + Y_VERIFY(CmsState->ClusterInfo->HasNode(id.NodeId)); + action.SetHost(CmsState->ClusterInfo->Node(id.NodeId).Host); + + if (reason) { + action.SetReason(reason); + } + + Send(CmsState->CmsActorId, std::move(ev)); + } + + void UpdateConfig() { + LOG_D("UpdateConfig"); + StartUpdater<TConfigUpdater>(ConfigUpdater); + } + + void OnConfigUpdated() { + LOG_D("Config was updated in " << (Now() - ConfigUpdater.StartedAt)); + + RemoveUntouched(); + *Counters->PDisksTotal = SentinelState->PDisks.size(); + + ScheduleUpdate<TEvSentinel::TEvUpdateConfig, TStateUpdater>( + ConfigUpdater, Config.UpdateConfigInterval, StateUpdater + ); + } + + void UpdateState() { + LOG_D("UpdateState"); + StartUpdater<TStateUpdater>(StateUpdater); + } + + void OnStateUpdated() { + LOG_D("State was updated in " << (Now() - StateUpdater.StartedAt)); + + EnsureAllTouched(); + + if (!CmsState->ClusterInfo) { + LOG_C("Missing cluster info"); + ScheduleUpdate<TEvSentinel::TEvUpdateState, TConfigUpdater>( + StateUpdater, Config.UpdateStateInterval, ConfigUpdater + ); + + return; + } + TClusterMap all(CmsState); - TGuardian changed(CmsState, Config.DataCenterRatio, Config.RoomRatio, Config.RackRatio); + TGuardian changed(CmsState, Config.DataCenterRatio, Config.RoomRatio, Config.RackRatio); TClusterMap::TPDiskIDSet alwaysAllowed; - - for (auto& pdisk : SentinelState->PDisks) { - const TPDiskID& id = pdisk.first; - TPDiskInfo& info = pdisk.second; - - if (!CmsState->ClusterInfo->HasNode(id.NodeId)) { - LOG_E("Missing node info" - << ": pdiskId# " << id); - continue; - } - - if (!CmsState->ClusterInfo->HasPDisk(id)) { - LOG_E("Missing pdisk info" - << ": pdiskId# " << id); - continue; - } - - all.AddPDisk(id); - if (info.IsChanged()) { + + for (auto& pdisk : SentinelState->PDisks) { + const TPDiskID& id = pdisk.first; + TPDiskInfo& info = pdisk.second; + + if (!CmsState->ClusterInfo->HasNode(id.NodeId)) { + LOG_E("Missing node info" + << ": pdiskId# " << id); + continue; + } + + if (!CmsState->ClusterInfo->HasPDisk(id)) { + LOG_E("Missing pdisk info" + << ": pdiskId# " << id); + continue; + } + + all.AddPDisk(id); + if (info.IsChanged()) { if (info.IsNewStatusGood()) { alwaysAllowed.insert(id); } else { changed.AddPDisk(id); } - } else { - info.AllowChanging(); - } - } - - TString issues; - THashSet<TPDiskID, TPDiskIDHash> disallowed; - + } else { + info.AllowChanging(); + } + } + + TString issues; + THashSet<TPDiskID, TPDiskIDHash> disallowed; + TClusterMap::TPDiskIDSet allowed = changed.GetAllowedPDisks(all, issues, disallowed); Copy(alwaysAllowed.begin(), alwaysAllowed.end(), std::inserter(allowed, allowed.begin())); for (const TPDiskID& id : allowed) { - Y_VERIFY(SentinelState->PDisks.contains(id)); - TPDiskInfo& info = SentinelState->PDisks.at(id); - - if (!info.IsChangingAllowed()) { - info.AllowChanging(); - continue; - } - - if (info.StatusChanger) { - continue; - } - - const EPDiskStatus status = info.GetStatus(); - TString reason; - info.ApplyChanges(reason); - const EPDiskStatus requiredStatus = info.GetStatus(); - - LOG_N("PDisk status changed" - << ": pdiskId# " << id - << ", status# " << status - << ", required status# " << requiredStatus - << ", reason# " << reason - << ", dry run# " << Config.DryRun); - LogStatusChange(id, status, requiredStatus, reason); - - if (!Config.DryRun) { - info.StatusChanger = Register(new TStatusChanger(SelfId(), CmsState, id, requiredStatus)); - (*Counters->PDisksPendingChange)++; - } - } - - for (const TPDiskID& id : disallowed) { - Y_VERIFY(SentinelState->PDisks.contains(id)); - SentinelState->PDisks.at(id).DisallowChanging(); - } - - if (issues) { - LOG_W(issues); - } - - ScheduleUpdate<TEvSentinel::TEvUpdateState, TConfigUpdater>( - StateUpdater, Config.UpdateStateInterval, ConfigUpdater - ); - } - - void Handle(TEvSentinel::TEvStatusChanged::TPtr& ev) { - const TPDiskID& id = ev->Get()->Id; - const bool success = ev->Get()->Success; - - LOG_D("Handle TEvSentinel::TEvStatusChanged" - << ": pdiskId# " << id - << ", success# " << (success ? "true" : "false")); - - auto it = SentinelState->PDisks.find(id); - if (it == SentinelState->PDisks.end()) { - LOG_W("Status of unknown pdisk has been changed" - << ": pdiskId# " << id); - return; - } - - if (!success) { - LOG_C("PDisk status has NOT been changed" - << ": pdiskId# " << id); - (*Counters->PDisksNotChanged)++; - } else { - LOG_N("PDisk status has been changed" - << ": pdiskId# " << id); - (*Counters->PDisksChanged)++; - } - + Y_VERIFY(SentinelState->PDisks.contains(id)); + TPDiskInfo& info = SentinelState->PDisks.at(id); + + if (!info.IsChangingAllowed()) { + info.AllowChanging(); + continue; + } + + if (info.StatusChanger) { + continue; + } + + const EPDiskStatus status = info.GetStatus(); + TString reason; + info.ApplyChanges(reason); + const EPDiskStatus requiredStatus = info.GetStatus(); + + LOG_N("PDisk status changed" + << ": pdiskId# " << id + << ", status# " << status + << ", required status# " << requiredStatus + << ", reason# " << reason + << ", dry run# " << Config.DryRun); + LogStatusChange(id, status, requiredStatus, reason); + + if (!Config.DryRun) { + info.StatusChanger = Register(new TStatusChanger(SelfId(), CmsState, id, requiredStatus)); + (*Counters->PDisksPendingChange)++; + } + } + + for (const TPDiskID& id : disallowed) { + Y_VERIFY(SentinelState->PDisks.contains(id)); + SentinelState->PDisks.at(id).DisallowChanging(); + } + + if (issues) { + LOG_W(issues); + } + + ScheduleUpdate<TEvSentinel::TEvUpdateState, TConfigUpdater>( + StateUpdater, Config.UpdateStateInterval, ConfigUpdater + ); + } + + void Handle(TEvSentinel::TEvStatusChanged::TPtr& ev) { + const TPDiskID& id = ev->Get()->Id; + const bool success = ev->Get()->Success; + + LOG_D("Handle TEvSentinel::TEvStatusChanged" + << ": pdiskId# " << id + << ", success# " << (success ? "true" : "false")); + + auto it = SentinelState->PDisks.find(id); + if (it == SentinelState->PDisks.end()) { + LOG_W("Status of unknown pdisk has been changed" + << ": pdiskId# " << id); + return; + } + + if (!success) { + LOG_C("PDisk status has NOT been changed" + << ": pdiskId# " << id); + (*Counters->PDisksNotChanged)++; + } else { + LOG_N("PDisk status has been changed" + << ": pdiskId# " << id); + (*Counters->PDisksChanged)++; + } + it->second.StatusChanger = TActorId(); - } - - void OnPipeDisconnected() { + } + + void OnPipeDisconnected() { if (const TActorId& actor = ConfigUpdater.Id) { - Send(actor, new TEvSentinel::TEvBSCPipeDisconnected()); - } - - for (const auto& pdisk : SentinelState->PDisks) { + Send(actor, new TEvSentinel::TEvBSCPipeDisconnected()); + } + + for (const auto& pdisk : SentinelState->PDisks) { if (const TActorId& actor = pdisk.second.StatusChanger) { - Send(actor, new TEvSentinel::TEvBSCPipeDisconnected()); - } - } - } - - void PassAway() override { + Send(actor, new TEvSentinel::TEvBSCPipeDisconnected()); + } + } + } + + void PassAway() override { if (const TActorId& actor = ConfigUpdater.Id) { - Send(actor, new TEvents::TEvPoisonPill()); - } - + Send(actor, new TEvents::TEvPoisonPill()); + } + if (const TActorId& actor = StateUpdater.Id) { - Send(actor, new TEvents::TEvPoisonPill()); - } - - for (const auto& pdisk : SentinelState->PDisks) { + Send(actor, new TEvents::TEvPoisonPill()); + } + + for (const auto& pdisk : SentinelState->PDisks) { if (const TActorId& actor = pdisk.second.StatusChanger) { - Send(actor, new TEvents::TEvPoisonPill()); - } - } - - TActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CMS_SENTINEL_ACTOR; - } - - static TStringBuf Name() { + Send(actor, new TEvents::TEvPoisonPill()); + } + } + + TActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CMS_SENTINEL_ACTOR; + } + + static TStringBuf Name() { return "Main"sv; - } - - explicit TSentinel(TCmsStatePtr cmsState) - : CmsState(cmsState) - , Config(CmsState->Config.SentinelConfig) - , SentinelState(new TSentinelState) - { - } - - void Bootstrap() { - auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "sentinel"); - Counters.Reset(new TCounters(counters)); - - UpdateConfig(); - UpdateState(); - - Become(&TThis::StateWork); - } - - STFUNC(StateWork) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - cFunc(TEvSentinel::TEvUpdateConfig::EventType, UpdateConfig); - cFunc(TEvSentinel::TEvConfigUpdated::EventType, OnConfigUpdated); - cFunc(TEvSentinel::TEvUpdateState::EventType, UpdateState); - cFunc(TEvSentinel::TEvStateUpdated::EventType, OnStateUpdated); - hFunc(TEvSentinel::TEvStatusChanged, Handle); - cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - TCmsStatePtr CmsState; - const TCmsSentinelConfig& Config; - THolder<TCounters> Counters; - - TUpdaterInfo ConfigUpdater; - TUpdaterInfo StateUpdater; - TSentinelState::TPtr SentinelState; - -}; // TSentinel - -} // NSentinel - -IActor* CreateSentinel(TCmsStatePtr state) { - return new NSentinel::TSentinel(state); -} - -} // NCms -} // NKikimr + } + + explicit TSentinel(TCmsStatePtr cmsState) + : CmsState(cmsState) + , Config(CmsState->Config.SentinelConfig) + , SentinelState(new TSentinelState) + { + } + + void Bootstrap() { + auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "sentinel"); + Counters.Reset(new TCounters(counters)); + + UpdateConfig(); + UpdateState(); + + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + cFunc(TEvSentinel::TEvUpdateConfig::EventType, UpdateConfig); + cFunc(TEvSentinel::TEvConfigUpdated::EventType, OnConfigUpdated); + cFunc(TEvSentinel::TEvUpdateState::EventType, UpdateState); + cFunc(TEvSentinel::TEvStateUpdated::EventType, OnStateUpdated); + hFunc(TEvSentinel::TEvStatusChanged, Handle); + cFunc(TEvSentinel::TEvBSCPipeDisconnected::EventType, OnPipeDisconnected); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + TCmsStatePtr CmsState; + const TCmsSentinelConfig& Config; + THolder<TCounters> Counters; + + TUpdaterInfo ConfigUpdater; + TUpdaterInfo StateUpdater; + TSentinelState::TPtr SentinelState; + +}; // TSentinel + +} // NSentinel + +IActor* CreateSentinel(TCmsStatePtr state) { + return new NSentinel::TSentinel(state); +} + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/sentinel.h b/ydb/core/cms/sentinel.h index d5b85f0c823..8fef2998bd7 100644 --- a/ydb/core/cms/sentinel.h +++ b/ydb/core/cms/sentinel.h @@ -1,57 +1,57 @@ -#pragma once - -#include "defs.h" -#include "cms_state.h" -#include "pdiskid.h" - -namespace NKikimr { -namespace NCms { - -struct TEvSentinel { - enum EEv { - EvUpdateConfig = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvConfigUpdated, - - EvUpdateState, - EvStateUpdated, - - EvStatusChanged, - - EvRetry, - EvTimeout, - - EvBSCPipeDisconnected, - - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); - - struct TEvUpdateConfig: public TEventLocal<TEvUpdateConfig, EvUpdateConfig> {}; - struct TEvConfigUpdated: public TEventLocal<TEvConfigUpdated, EvConfigUpdated> {}; - - struct TEvUpdateState: public TEventLocal<TEvUpdateState, EvUpdateState> {}; - struct TEvStateUpdated: public TEventLocal<TEvStateUpdated, EvStateUpdated> {}; - - struct TEvRetry: public TEventLocal<TEvRetry, EvRetry> {}; - struct TEvTimeout: public TEventLocal<TEvTimeout, EvTimeout> {}; - - struct TEvStatusChanged: public TEventLocal<TEvStatusChanged, EvStatusChanged> { - TPDiskID Id; - bool Success; - - explicit TEvStatusChanged(const TPDiskID& id, bool success) - : Id(id) - , Success(success) - { - } - }; - - struct TEvBSCPipeDisconnected: public TEventLocal<TEvBSCPipeDisconnected, EvBSCPipeDisconnected> {}; - -}; // TEvSentinel - -IActor* CreateSentinel(TCmsStatePtr state); - -} // NCms -} // NKikimr +#pragma once + +#include "defs.h" +#include "cms_state.h" +#include "pdiskid.h" + +namespace NKikimr { +namespace NCms { + +struct TEvSentinel { + enum EEv { + EvUpdateConfig = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvConfigUpdated, + + EvUpdateState, + EvStateUpdated, + + EvStatusChanged, + + EvRetry, + EvTimeout, + + EvBSCPipeDisconnected, + + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + struct TEvUpdateConfig: public TEventLocal<TEvUpdateConfig, EvUpdateConfig> {}; + struct TEvConfigUpdated: public TEventLocal<TEvConfigUpdated, EvConfigUpdated> {}; + + struct TEvUpdateState: public TEventLocal<TEvUpdateState, EvUpdateState> {}; + struct TEvStateUpdated: public TEventLocal<TEvStateUpdated, EvStateUpdated> {}; + + struct TEvRetry: public TEventLocal<TEvRetry, EvRetry> {}; + struct TEvTimeout: public TEventLocal<TEvTimeout, EvTimeout> {}; + + struct TEvStatusChanged: public TEventLocal<TEvStatusChanged, EvStatusChanged> { + TPDiskID Id; + bool Success; + + explicit TEvStatusChanged(const TPDiskID& id, bool success) + : Id(id) + , Success(success) + { + } + }; + + struct TEvBSCPipeDisconnected: public TEventLocal<TEvBSCPipeDisconnected, EvBSCPipeDisconnected> {}; + +}; // TEvSentinel + +IActor* CreateSentinel(TCmsStatePtr state); + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/sentinel_impl.h b/ydb/core/cms/sentinel_impl.h index f563cc5f60a..8aa6934bafa 100644 --- a/ydb/core/cms/sentinel_impl.h +++ b/ydb/core/cms/sentinel_impl.h @@ -1,84 +1,84 @@ -#pragma once - -#include "defs.h" -#include "cms_state.h" -#include "pdiskid.h" -#include "pdisk_state.h" - +#pragma once + +#include "defs.h" +#include "cms_state.h" +#include "pdiskid.h" +#include "pdisk_state.h" + #include <ydb/core/protos/blobstorage_config.pb.h> - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/map.h> - -namespace NKikimr { -namespace NCms { -namespace NSentinel { - -using EPDiskStatus = NKikimrBlobStorage::EDriveStatus; -using TLimitsMap = TMap<EPDiskState, ui32>; - -class TPDiskStatusComputer { -public: - explicit TPDiskStatusComputer(const ui32& defaultStateLimit, const TLimitsMap& stateLimits); - - void AddState(EPDiskState state); - EPDiskStatus Compute(EPDiskStatus current, TString& reason) const; - void Reset(); - -private: - const ui32& DefaultStateLimit; - const TLimitsMap& StateLimits; - + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/map.h> + +namespace NKikimr { +namespace NCms { +namespace NSentinel { + +using EPDiskStatus = NKikimrBlobStorage::EDriveStatus; +using TLimitsMap = TMap<EPDiskState, ui32>; + +class TPDiskStatusComputer { +public: + explicit TPDiskStatusComputer(const ui32& defaultStateLimit, const TLimitsMap& stateLimits); + + void AddState(EPDiskState state); + EPDiskStatus Compute(EPDiskStatus current, TString& reason) const; + void Reset(); + +private: + const ui32& DefaultStateLimit; + const TLimitsMap& StateLimits; + EPDiskState State = NKikimrBlobStorage::TPDiskState::Unknown; mutable EPDiskState PrevState = State; - ui64 StateCounter; - -}; // TPDiskStatusComputer - -class TPDiskStatus: private TPDiskStatusComputer { -public: - explicit TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits); - - void AddState(EPDiskState state); - bool IsChanged(TString& reason) const; - bool IsChanged() const; - void ApplyChanges(TString& reason); - void ApplyChanges(); - EPDiskStatus GetStatus() const; + ui64 StateCounter; + +}; // TPDiskStatusComputer + +class TPDiskStatus: private TPDiskStatusComputer { +public: + explicit TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits); + + void AddState(EPDiskState state); + bool IsChanged(TString& reason) const; + bool IsChanged() const; + void ApplyChanges(TString& reason); + void ApplyChanges(); + EPDiskStatus GetStatus() const; bool IsNewStatusGood() const; - - bool IsChangingAllowed() const; - void AllowChanging(); - void DisallowChanging(); - -private: - EPDiskStatus Current; - bool ChangingAllowed; - -}; // TPDiskStatus - -struct TPDiskInfo: public TPDiskStatus { + + bool IsChangingAllowed() const; + void AllowChanging(); + void DisallowChanging(); + +private: + EPDiskStatus Current; + bool ChangingAllowed; + +}; // TPDiskStatus + +struct TPDiskInfo: public TPDiskStatus { TActorId StatusChanger; - - explicit TPDiskInfo(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits); - - bool IsTouched() const { return Touched; } - void Touch() { Touched = true; } - void ClearTouched() { Touched = false; } - - void AddState(EPDiskState state); - -private: - bool Touched; - -}; // TPDiskInfo - + + explicit TPDiskInfo(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits); + + bool IsTouched() const { return Touched; } + void Touch() { Touched = true; } + void ClearTouched() { Touched = false; } + + void AddState(EPDiskState state); + +private: + bool Touched; + +}; // TPDiskInfo + class TClusterMap { public: - using TPDiskIDSet = THashSet<TPDiskID, TPDiskIDHash>; + using TPDiskIDSet = THashSet<TPDiskID, TPDiskIDHash>; using TDistribution = THashMap<TString, TPDiskIDSet>; - + TCmsStatePtr State; TDistribution ByDataCenter; TDistribution ByRoom; @@ -90,25 +90,25 @@ public: }; // TClusterMap class TGuardian : public TClusterMap { - static bool CheckRatio(ui32 check, ui32 base, ui32 ratio) { - return (check * 100) <= (base * ratio); - } - - static bool CheckRatio(const TDistribution::value_type& check, const TDistribution& base, ui32 ratio) { - return CheckRatio(check.second.size(), base.at(check.first).size(), ratio); - } - -public: - explicit TGuardian(TCmsStatePtr state, ui32 dataCenterRatio = 100, ui32 roomRatio = 100, ui32 rackRatio = 100); - + static bool CheckRatio(ui32 check, ui32 base, ui32 ratio) { + return (check * 100) <= (base * ratio); + } + + static bool CheckRatio(const TDistribution::value_type& check, const TDistribution& base, ui32 ratio) { + return CheckRatio(check.second.size(), base.at(check.first).size(), ratio); + } + +public: + explicit TGuardian(TCmsStatePtr state, ui32 dataCenterRatio = 100, ui32 roomRatio = 100, ui32 rackRatio = 100); + TPDiskIDSet GetAllowedPDisks(const TClusterMap& all, TString& issues, TPDiskIDSet& disallowed) const; - -private: - const ui32 DataCenterRatio; - const ui32 RoomRatio; - const ui32 RackRatio; -}; // TGuardian - -} // NSentinel -} // NCms -} // NKikimr + +private: + const ui32 DataCenterRatio; + const ui32 RoomRatio; + const ui32 RackRatio; +}; // TGuardian + +} // NSentinel +} // NCms +} // NKikimr diff --git a/ydb/core/cms/sentinel_ut.cpp b/ydb/core/cms/sentinel_ut.cpp index 545fc9aec71..62c0fdf309d 100644 --- a/ydb/core/cms/sentinel_ut.cpp +++ b/ydb/core/cms/sentinel_ut.cpp @@ -1,23 +1,23 @@ -#include "cms_ut_common.h" -#include "sentinel.h" -#include "sentinel_impl.h" - +#include "cms_ut_common.h" +#include "sentinel.h" +#include "sentinel_impl.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/hash_set.h> -#include <util/generic/vector.h> -#include <util/generic/xrange.h> -#include <util/random/random.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NCmsTest { - -static constexpr ui32 DefaultStateLimit = 5; + +#include <util/generic/hash_set.h> +#include <util/generic/vector.h> +#include <util/generic/xrange.h> +#include <util/random/random.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NCmsTest { + +static constexpr ui32 DefaultStateLimit = 5; static constexpr ui32 DefaultErrorStateLimit = 60; -auto DefaultStateLimits = NCms::TCmsSentinelConfig::DefaultStateLimits(); - -static constexpr NCms::EPDiskState ErrorStates[] = { +auto DefaultStateLimits = NCms::TCmsSentinelConfig::DefaultStateLimits(); + +static constexpr NCms::EPDiskState ErrorStates[] = { NKikimrBlobStorage::TPDiskState::InitialFormatReadError, NKikimrBlobStorage::TPDiskState::InitialSysLogReadError, NKikimrBlobStorage::TPDiskState::InitialSysLogParseError, @@ -27,58 +27,58 @@ static constexpr NCms::EPDiskState ErrorStates[] = { NKikimrBlobStorage::TPDiskState::OpenFileError, NKikimrBlobStorage::TPDiskState::ChunkQuotaError, NKikimrBlobStorage::TPDiskState::DeviceIoError, -}; - -constexpr NCms::EPDiskState FaultyStates[] = { +}; + +constexpr NCms::EPDiskState FaultyStates[] = { NKikimrBlobStorage::TPDiskState::Initial, NKikimrBlobStorage::TPDiskState::InitialFormatRead, NKikimrBlobStorage::TPDiskState::InitialSysLogRead, NKikimrBlobStorage::TPDiskState::InitialCommonLogRead, -}; - -Y_UNIT_TEST_SUITE(TSentinelBaseTests) { - - using namespace NCms; - using namespace NCms::NSentinel; - using TPDiskID = NCms::TPDiskID; - - Y_UNIT_TEST(PDiskInitialStatus) { - const EPDiskStatus AllStatuses[] = { - EPDiskStatus::UNKNOWN, - EPDiskStatus::ACTIVE, - EPDiskStatus::INACTIVE, - EPDiskStatus::BROKEN, - EPDiskStatus::SPARE, - EPDiskStatus::FAULTY, - }; - - for (const EPDiskStatus status : AllStatuses) { - TPDiskStatus st(status, DefaultStateLimit, DefaultStateLimits); - - UNIT_ASSERT(!st.IsChanged()); - UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), status); - } - } - - Y_UNIT_TEST(PDiskErrorState) { - for (const EPDiskState state : ErrorStates) { - const EPDiskStatus initialStatus = EPDiskStatus::ACTIVE; - TPDiskStatus st(initialStatus, DefaultStateLimit, DefaultStateLimits); - +}; + +Y_UNIT_TEST_SUITE(TSentinelBaseTests) { + + using namespace NCms; + using namespace NCms::NSentinel; + using TPDiskID = NCms::TPDiskID; + + Y_UNIT_TEST(PDiskInitialStatus) { + const EPDiskStatus AllStatuses[] = { + EPDiskStatus::UNKNOWN, + EPDiskStatus::ACTIVE, + EPDiskStatus::INACTIVE, + EPDiskStatus::BROKEN, + EPDiskStatus::SPARE, + EPDiskStatus::FAULTY, + }; + + for (const EPDiskStatus status : AllStatuses) { + TPDiskStatus st(status, DefaultStateLimit, DefaultStateLimits); + + UNIT_ASSERT(!st.IsChanged()); + UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), status); + } + } + + Y_UNIT_TEST(PDiskErrorState) { + for (const EPDiskState state : ErrorStates) { + const EPDiskStatus initialStatus = EPDiskStatus::ACTIVE; + TPDiskStatus st(initialStatus, DefaultStateLimit, DefaultStateLimits); + for (ui32 i = 1; i < DefaultStateLimits[state]; ++i) { st.AddState(state); UNIT_ASSERT(!st.IsChanged()); UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), initialStatus); } - st.AddState(state); - UNIT_ASSERT(st.IsChanged()); - - st.ApplyChanges(); - UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), EPDiskStatus::FAULTY); - } - } - + st.AddState(state); + UNIT_ASSERT(st.IsChanged()); + + st.ApplyChanges(); + UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), EPDiskStatus::FAULTY); + } + } + Y_UNIT_TEST(PDiskInactiveAfterStateChange) { for (const EPDiskState state : ErrorStates) { const EPDiskStatus initialStatus = EPDiskStatus::ACTIVE; @@ -115,37 +115,37 @@ Y_UNIT_TEST_SUITE(TSentinelBaseTests) { } } - Y_UNIT_TEST(PDiskFaultyState) { - for (const EPDiskState state : FaultyStates) { - const EPDiskStatus initialStatus = EPDiskStatus::ACTIVE; - TPDiskStatus st(initialStatus, DefaultStateLimit, DefaultStateLimits); - - for (ui32 i = 1; i < DefaultStateLimit; ++i) { - st.AddState(state); - - UNIT_ASSERT(!st.IsChanged()); - UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), initialStatus); - } - - st.AddState(state); - - UNIT_ASSERT(st.IsChanged()); - - st.ApplyChanges(); - UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), EPDiskStatus::FAULTY); - } - } - + Y_UNIT_TEST(PDiskFaultyState) { + for (const EPDiskState state : FaultyStates) { + const EPDiskStatus initialStatus = EPDiskStatus::ACTIVE; + TPDiskStatus st(initialStatus, DefaultStateLimit, DefaultStateLimits); + + for (ui32 i = 1; i < DefaultStateLimit; ++i) { + st.AddState(state); + + UNIT_ASSERT(!st.IsChanged()); + UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), initialStatus); + } + + st.AddState(state); + + UNIT_ASSERT(st.IsChanged()); + + st.ApplyChanges(); + UNIT_ASSERT_VALUES_EQUAL(st.GetStatus(), EPDiskStatus::FAULTY); + } + } + TCmsStatePtr MockCmsState(ui16 numDataCenter, ui16 racksPerDataCenter, ui16 nodesPerRack, bool anyDC, bool anyRack) { - TCmsStatePtr state = new TCmsState; - state->ClusterInfo = new TClusterInfo; - + TCmsStatePtr state = new TCmsState; + state->ClusterInfo = new TClusterInfo; + for (ui64 dc : xrange(numDataCenter)) { for (ui64 rack : xrange(racksPerDataCenter)) { for (ui64 node : xrange(nodesPerRack)) { const ui64 id = (dc << 32) | (rack << 16) | node; const TString name = TStringBuilder() << "dc_" << dc << "-rack_" << rack << "-node_" << node; - + NActorsInterconnect::TNodeLocation location; if (!anyDC) { location.SetDataCenter(ToString(dc + 1)); @@ -154,8 +154,8 @@ Y_UNIT_TEST_SUITE(TSentinelBaseTests) { location.SetRack(ToString(rack + 1)); } location.SetUnit(ToString(id)); - - state->ClusterInfo->AddNode(TEvInterconnect::TNodeInfo(id, name, name, name, 10000, TNodeLocation(location)), nullptr); + + state->ClusterInfo->AddNode(TEvInterconnect::TNodeInfo(id, name, name, name, 10000, TNodeLocation(location)), nullptr); NKikimrBlobStorage::TBaseConfig::TPDisk pdisk; pdisk.SetNodeId(id); @@ -163,72 +163,72 @@ Y_UNIT_TEST_SUITE(TSentinelBaseTests) { pdisk.SetPath("pdisk.data"); state->ClusterInfo->AddPDisk(pdisk); } - } - } - - return state; - } - - void GuardianDataCenterRatio(ui16 numDataCenter, const TVector<ui16>& nodesPerDataCenterVariants, bool anyDC = false) { - UNIT_ASSERT(!anyDC || numDataCenter == 1); - - for (ui16 nodesPerDataCenter : nodesPerDataCenterVariants) { + } + } + + return state; + } + + void GuardianDataCenterRatio(ui16 numDataCenter, const TVector<ui16>& nodesPerDataCenterVariants, bool anyDC = false) { + UNIT_ASSERT(!anyDC || numDataCenter == 1); + + for (ui16 nodesPerDataCenter : nodesPerDataCenterVariants) { TCmsStatePtr state = MockCmsState(numDataCenter, nodesPerDataCenter, 1, anyDC, false); - TGuardian all(state); - TGuardian changed(state, 50); - THashSet<TPDiskID, TPDiskIDHash> changedSet; - - const auto& nodes = state->ClusterInfo->AllNodes(); - - TVector<ui32> changedCount(numDataCenter); - for (const auto& node : nodes) { + TGuardian all(state); + TGuardian changed(state, 50); + THashSet<TPDiskID, TPDiskIDHash> changedSet; + + const auto& nodes = state->ClusterInfo->AllNodes(); + + TVector<ui32> changedCount(numDataCenter); + for (const auto& node : nodes) { const ui64 nodeId = node.second->NodeId; - const TPDiskID id(nodeId, 0); - - all.AddPDisk(id); + const TPDiskID id(nodeId, 0); + + all.AddPDisk(id); if (changedCount[nodeId >> 32]++ < (nodesPerDataCenter / 2)) { - changed.AddPDisk(id); - changedSet.insert(id); - } - } - - TString issues; - THashSet<TPDiskID, TPDiskIDHash> disallowed; - - UNIT_ASSERT_VALUES_EQUAL(changed.GetAllowedPDisks(all, issues, disallowed), changedSet); - UNIT_ASSERT(disallowed.empty()); - UNIT_ASSERT(issues.empty()); - - changedCount.assign(numDataCenter, 0); - for (const auto& node : nodes) { + changed.AddPDisk(id); + changedSet.insert(id); + } + } + + TString issues; + THashSet<TPDiskID, TPDiskIDHash> disallowed; + + UNIT_ASSERT_VALUES_EQUAL(changed.GetAllowedPDisks(all, issues, disallowed), changedSet); + UNIT_ASSERT(disallowed.empty()); + UNIT_ASSERT(issues.empty()); + + changedCount.assign(numDataCenter, 0); + for (const auto& node : nodes) { const ui64 nodeId = node.second->NodeId; - const TPDiskID id(nodeId, 0); - + const TPDiskID id(nodeId, 0); + if (changedCount[nodeId >> 32]++ < ((nodesPerDataCenter / 2) + 1)) { - changed.AddPDisk(id); - changedSet.insert(id); - } - } - - disallowed.clear(); - if (!anyDC) { - UNIT_ASSERT(changed.GetAllowedPDisks(all, issues, disallowed).empty()); - UNIT_ASSERT_VALUES_EQUAL(disallowed, changedSet); - UNIT_ASSERT_STRING_CONTAINS(issues, "due to DataCenterRatio"); - } else { - UNIT_ASSERT_VALUES_EQUAL(changed.GetAllowedPDisks(all, issues, disallowed), changedSet); - UNIT_ASSERT(disallowed.empty()); - UNIT_ASSERT(issues.empty()); - } - } - } - - Y_UNIT_TEST(GuardianDataCenterRatio) { - GuardianDataCenterRatio(1, {3, 4, 5}); - GuardianDataCenterRatio(3, {3, 4, 5}); - GuardianDataCenterRatio(1, {3, 4, 5}, true); - } - + changed.AddPDisk(id); + changedSet.insert(id); + } + } + + disallowed.clear(); + if (!anyDC) { + UNIT_ASSERT(changed.GetAllowedPDisks(all, issues, disallowed).empty()); + UNIT_ASSERT_VALUES_EQUAL(disallowed, changedSet); + UNIT_ASSERT_STRING_CONTAINS(issues, "due to DataCenterRatio"); + } else { + UNIT_ASSERT_VALUES_EQUAL(changed.GetAllowedPDisks(all, issues, disallowed), changedSet); + UNIT_ASSERT(disallowed.empty()); + UNIT_ASSERT(issues.empty()); + } + } + } + + Y_UNIT_TEST(GuardianDataCenterRatio) { + GuardianDataCenterRatio(1, {3, 4, 5}); + GuardianDataCenterRatio(3, {3, 4, 5}); + GuardianDataCenterRatio(1, {3, 4, 5}, true); + } + void GuardianRackRatio(ui16 numRacks, const TVector<ui16>& nodesPerRackVariants, bool anyRack) { for (ui16 nodesPerRack : nodesPerRackVariants) { TCmsStatePtr state = MockCmsState(1, numRacks, nodesPerRack, false, anyRack); @@ -294,98 +294,98 @@ Y_UNIT_TEST_SUITE(TSentinelBaseTests) { } -} // TSentinelBaseTests - -Y_UNIT_TEST_SUITE(TSentinelTests) { - - using namespace NCms; - using namespace NCms::NSentinel; - using TPDiskID = NCms::TPDiskID; - - const auto& MockConfig = TFakeNodeWhiteboardService::Config; - auto& MockNodes = TFakeNodeWhiteboardService::Info; - - class TTestEnv: public TCmsTestEnv { - static void MockClusterInfo(TClusterInfoPtr& info) { - info.Reset(new TClusterInfo); - - for (const auto& pdisk : MockConfig.GetResponse().GetStatus(0).GetBaseConfig().GetPDisk()) { - const ui32 nodeId = pdisk.GetNodeId(); - - if (info->HasNode(nodeId)) { - info->AddPDisk(pdisk); - continue; - } - - const TString name = TStringBuilder() << "node-" << nodeId; +} // TSentinelBaseTests + +Y_UNIT_TEST_SUITE(TSentinelTests) { + + using namespace NCms; + using namespace NCms::NSentinel; + using TPDiskID = NCms::TPDiskID; + + const auto& MockConfig = TFakeNodeWhiteboardService::Config; + auto& MockNodes = TFakeNodeWhiteboardService::Info; + + class TTestEnv: public TCmsTestEnv { + static void MockClusterInfo(TClusterInfoPtr& info) { + info.Reset(new TClusterInfo); + + for (const auto& pdisk : MockConfig.GetResponse().GetStatus(0).GetBaseConfig().GetPDisk()) { + const ui32 nodeId = pdisk.GetNodeId(); + + if (info->HasNode(nodeId)) { + info->AddPDisk(pdisk); + continue; + } + + const TString name = TStringBuilder() << "node-" << nodeId; NActorsInterconnect::TNodeLocation location; location.SetRack(TStringBuilder() << "rack-" << (nodeId - 1) % 8 + 1); - info->AddNode(TEvInterconnect::TNodeInfo(nodeId, name, name, name, 10000, TNodeLocation(location)), nullptr); - info->AddPDisk(pdisk); - } - } - - void WaitForSentinelBoot() { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvSentinel::TEvConfigUpdated::EventType); - options.FinalEvents.emplace_back(TEvSentinel::TEvStateUpdated::EventType); - UNIT_ASSERT(DispatchEvents(options)); - } - + info->AddNode(TEvInterconnect::TNodeInfo(nodeId, name, name, name, 10000, TNodeLocation(location)), nullptr); + info->AddPDisk(pdisk); + } + } + + void WaitForSentinelBoot() { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvSentinel::TEvConfigUpdated::EventType); + options.FinalEvents.emplace_back(TEvSentinel::TEvStateUpdated::EventType); + UNIT_ASSERT(DispatchEvents(options)); + } + void SetPDiskStateImpl(const TSet<TPDiskID>& ids, EPDiskState state) { for (const auto& id : ids) { Y_VERIFY(MockNodes.contains(id.NodeId)); auto& node = MockNodes.at(id.NodeId); - + Y_VERIFY(node.PDiskStateInfo.contains(id.DiskId)); auto& pdisk = node.PDiskStateInfo.at(id.DiskId); - + pdisk.SetState(state); } Send(new IEventHandle(Sentinel, TActorId(), new TEvSentinel::TEvUpdateState)); - } - - public: - explicit TTestEnv(ui32 nodeCount, ui32 pdisks) - : TCmsTestEnv(nodeCount, pdisks) - { - SetScheduledEventFilter([this](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev, TDuration, TInstant&) -> bool { - if (ev->Recipient != Sentinel) { - return true; - } - - switch (ev->GetTypeRewrite()) { - case TEvSentinel::TEvUpdateConfig::EventType: - case TEvSentinel::TEvUpdateState::EventType: - return false; - - default: - return true; - } - }); - - State = new TCmsState; - MockClusterInfo(State->ClusterInfo); - - Sentinel = Register(CreateSentinel(State)); - EnableScheduleForActor(Sentinel, true); - WaitForSentinelBoot(); - + } + + public: + explicit TTestEnv(ui32 nodeCount, ui32 pdisks) + : TCmsTestEnv(nodeCount, pdisks) + { + SetScheduledEventFilter([this](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev, TDuration, TInstant&) -> bool { + if (ev->Recipient != Sentinel) { + return true; + } + + switch (ev->GetTypeRewrite()) { + case TEvSentinel::TEvUpdateConfig::EventType: + case TEvSentinel::TEvUpdateState::EventType: + return false; + + default: + return true; + } + }); + + State = new TCmsState; + MockClusterInfo(State->ClusterInfo); + + Sentinel = Register(CreateSentinel(State)); + EnableScheduleForActor(Sentinel, true); + WaitForSentinelBoot(); + SetLogPriority(NKikimrServices::CMS, NLog::PRI_DEBUG); - } - - TPDiskID RandomPDiskID() const { - const auto& config = MockConfig.GetResponse().GetStatus(0).GetBaseConfig(); - const auto& pdisk = config.GetPDisk(RandomNumber(config.PDiskSize() - 1)); - return TPDiskID(pdisk.GetNodeId(), pdisk.GetPDiskId()); - } - + } + + TPDiskID RandomPDiskID() const { + const auto& config = MockConfig.GetResponse().GetStatus(0).GetBaseConfig(); + const auto& pdisk = config.GetPDisk(RandomNumber(config.PDiskSize() - 1)); + return TPDiskID(pdisk.GetNodeId(), pdisk.GetPDiskId()); + } + TSet<TPDiskID> PDisksForRandomRack() const { auto nodes = State->ClusterInfo->AllNodes(); size_t idx = RandomNumber(nodes.size() - 1); auto target = std::next(nodes.begin(), idx)->second; - + TString targetRack = target->Location.HasKey(TNodeLocation::TKeys::Rack) ? target->Location.GetRackId() : ""; @@ -414,16 +414,16 @@ Y_UNIT_TEST_SUITE(TSentinelTests) { void SetPDiskState(const TSet<TPDiskID>& pdisks, EPDiskState state) { SetPDiskStateImpl(pdisks, state); - TDispatchOptions options; + TDispatchOptions options; options.FinalEvents.emplace_back(TEvSentinel::TEvStateUpdated::EventType, 1); - UNIT_ASSERT(DispatchEvents(options)); - } - + UNIT_ASSERT(DispatchEvents(options)); + } + void SetPDiskState(const TSet<TPDiskID>& pdisks, EPDiskState state, EPDiskStatus expectedStatus) { SetPDiskStateImpl(pdisks, state); - - bool stateUpdated = false; - + + bool stateUpdated = false; + struct TPDiskUpdates { bool UpdateStatusRequested = false; bool StatusChanged = false; @@ -433,44 +433,44 @@ Y_UNIT_TEST_SUITE(TSentinelTests) { pdiskUpdates[id] = {};// TPDiskUpdates {false, false}); } - auto check = [&](IEventHandle& ev) { - switch (ev.GetTypeRewrite()) { - case TEvSentinel::TEvStateUpdated::EventType: - stateUpdated = true; - break; - - case TEvBlobStorage::TEvControllerConfigRequest::EventType: + auto check = [&](IEventHandle& ev) { + switch (ev.GetTypeRewrite()) { + case TEvSentinel::TEvStateUpdated::EventType: + stateUpdated = true; + break; + + case TEvBlobStorage::TEvControllerConfigRequest::EventType: { - const auto& request = ev.Get<TEvBlobStorage::TEvControllerConfigRequest>()->Record; - - UNIT_ASSERT(request.HasRequest()); - if (request.GetRequest().CommandSize() && request.GetRequest().GetCommand(0).HasUpdateDriveStatus()) { + const auto& request = ev.Get<TEvBlobStorage::TEvControllerConfigRequest>()->Record; + + UNIT_ASSERT(request.HasRequest()); + if (request.GetRequest().CommandSize() && request.GetRequest().GetCommand(0).HasUpdateDriveStatus()) { const auto& update = request.GetRequest().GetCommand(0).GetUpdateDriveStatus(); TPDiskID id(update.GetHostKey().GetNodeId(), update.GetPDiskId()); auto it = pdiskUpdates.find(id); if (it != pdiskUpdates.end()) { it->second.UpdateStatusRequested = (expectedStatus == update.GetStatus()); } - } - } - break; - - case TEvSentinel::TEvStatusChanged::EventType: + } + } + break; + + case TEvSentinel::TEvStatusChanged::EventType: { - const auto* event = ev.Get<TEvSentinel::TEvStatusChanged>(); - + const auto* event = ev.Get<TEvSentinel::TEvStatusChanged>(); + auto it = pdiskUpdates.find(event->Id); if (it != pdiskUpdates.end()) { - UNIT_ASSERT(event->Success); + UNIT_ASSERT(event->Success); it->second.StatusChanged = true; - } - } - break; - - default: - break; - } - + } + } + break; + + default: + break; + } + bool allUpdateStatusRequested = true; bool allStatusChanged = true; for (const auto& [id, info] : pdiskUpdates) { @@ -478,64 +478,64 @@ Y_UNIT_TEST_SUITE(TSentinelTests) { allStatusChanged &= info.StatusChanged; } return stateUpdated && pdiskUpdates.size() && allUpdateStatusRequested && allStatusChanged; - }; - - TDispatchOptions options; - options.FinalEvents.emplace_back(check); - UNIT_ASSERT(DispatchEvents(options)); - } - - private: - TCmsStatePtr State; + }; + + TDispatchOptions options; + options.FinalEvents.emplace_back(check); + UNIT_ASSERT(DispatchEvents(options)); + } + + private: + TCmsStatePtr State; TActorId Sentinel; - - }; // TTestEnv - - Y_UNIT_TEST(Smoke) { - TTestEnv env(8, 4); - } - - Y_UNIT_TEST(PDiskUnknownState) { - TTestEnv env(8, 4); - - const auto reservedStates = TVector<EPDiskState>{ + + }; // TTestEnv + + Y_UNIT_TEST(Smoke) { + TTestEnv env(8, 4); + } + + Y_UNIT_TEST(PDiskUnknownState) { + TTestEnv env(8, 4); + + const auto reservedStates = TVector<EPDiskState>{ NKikimrBlobStorage::TPDiskState::Reserved14, NKikimrBlobStorage::TPDiskState::Reserved15, NKikimrBlobStorage::TPDiskState::Reserved16, - }; - - for (const auto state : reservedStates) { - const TPDiskID id = env.RandomPDiskID(); + }; + + for (const auto state : reservedStates) { + const TPDiskID id = env.RandomPDiskID(); env.SetPDiskState({id}, state); - } - } - - Y_UNIT_TEST(PDiskErrorState) { - TTestEnv env(8, 4); - - for (const EPDiskState state : ErrorStates) { - const TPDiskID id = env.RandomPDiskID(); - + } + } + + Y_UNIT_TEST(PDiskErrorState) { + TTestEnv env(8, 4); + + for (const EPDiskState state : ErrorStates) { + const TPDiskID id = env.RandomPDiskID(); + env.SetPDiskState({id}, state, EPDiskStatus::FAULTY); env.SetPDiskState({id}, NKikimrBlobStorage::TPDiskState::Normal, EPDiskStatus::ACTIVE); - } - } - - Y_UNIT_TEST(PDiskFaultyState) { - TTestEnv env(8, 4); - - for (const EPDiskState state : FaultyStates) { - const TPDiskID id = env.RandomPDiskID(); - - for (ui32 i = 1; i < DefaultStateLimit; ++i) { + } + } + + Y_UNIT_TEST(PDiskFaultyState) { + TTestEnv env(8, 4); + + for (const EPDiskState state : FaultyStates) { + const TPDiskID id = env.RandomPDiskID(); + + for (ui32 i = 1; i < DefaultStateLimit; ++i) { env.SetPDiskState({id}, state); - } - + } + env.SetPDiskState({id}, state, EPDiskStatus::FAULTY); env.SetPDiskState({id}, NKikimrBlobStorage::TPDiskState::Normal, EPDiskStatus::ACTIVE); - } - } - + } + } + Y_UNIT_TEST(PDiskRackGuardHalfRack) { TTestEnv env(16, 4); // 16 nodes are distributed into 8 racks, 2 per rack @@ -575,7 +575,7 @@ Y_UNIT_TEST_SUITE(TSentinelTests) { env.SetPDiskState(pdisks, NKikimrBlobStorage::TPDiskState::Normal, EPDiskStatus::ACTIVE); } } -} // TSentinelTests - -} // NCmsTest -} // NKikimr +} // TSentinelTests + +} // NCmsTest +} // NKikimr diff --git a/ydb/core/cms/services.cpp b/ydb/core/cms/services.cpp index 7d8136aed57..78824908f02 100644 --- a/ydb/core/cms/services.cpp +++ b/ydb/core/cms/services.cpp @@ -1,24 +1,24 @@ -#include "services.h" - -#include <util/generic/hash.h> - -namespace NKikimr { -namespace NCms { - -bool TryFromWhiteBoardRole(const TString& role, EService& value) { - static const THashMap<TString, EService> roleToService = { - {"Storage", EService::Storage}, - {"Tenant", EService::DynamicNode}, - }; - - auto it = roleToService.find(role); - if (it == roleToService.end()) { - return false; - } - - value = it->second; - return true; -} - -} // NCms -} // NKikimr +#include "services.h" + +#include <util/generic/hash.h> + +namespace NKikimr { +namespace NCms { + +bool TryFromWhiteBoardRole(const TString& role, EService& value) { + static const THashMap<TString, EService> roleToService = { + {"Storage", EService::Storage}, + {"Tenant", EService::DynamicNode}, + }; + + auto it = roleToService.find(role); + if (it == roleToService.end()) { + return false; + } + + value = it->second; + return true; +} + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/services.h b/ydb/core/cms/services.h index b15cd29b026..2e3467f2324 100644 --- a/ydb/core/cms/services.h +++ b/ydb/core/cms/services.h @@ -1,19 +1,19 @@ -#pragma once - -#include <util/generic/flags.h> - -namespace NKikimr { -namespace NCms { - -enum class EService: ui8 { - Storage = 1 /* "storage" */, - DynamicNode = 2 /* "dynnode" */, -}; - -Y_DECLARE_FLAGS(TServices, EService); -Y_DECLARE_OPERATORS_FOR_FLAGS(TServices); - -bool TryFromWhiteBoardRole(const TString& role, EService& value); - -} // NCms -} // NKikimr +#pragma once + +#include <util/generic/flags.h> + +namespace NKikimr { +namespace NCms { + +enum class EService: ui8 { + Storage = 1 /* "storage" */, + DynamicNode = 2 /* "dynnode" */, +}; + +Y_DECLARE_FLAGS(TServices, EService); +Y_DECLARE_OPERATORS_FOR_FLAGS(TServices); + +bool TryFromWhiteBoardRole(const TString& role, EService& value); + +} // NCms +} // NKikimr diff --git a/ydb/core/cms/ui/config_forms.js b/ydb/core/cms/ui/config_forms.js index 633c872a3ed..27518ef5ca1 100644 --- a/ydb/core/cms/ui/config_forms.js +++ b/ydb/core/cms/ui/config_forms.js @@ -86,7 +86,7 @@ function formatCommonItemText(item) { function addEditButtons(div, item) { var buttonsHTML = ` <div style="z-index: 1; position: absolute; top: 0; right: 0; margin: 5px;"> - <button data-action="edit" class="${item.kind.editable ? '' : 'd-none'} btn-cfg-item cfg-item-edit-icon"></button> + <button data-action="edit" class="${item.kind.editable ? '' : 'd-none'} btn-cfg-item cfg-item-edit-icon"></button> <button data-action="remove" class="btn-cfg-item cfg-item-remove-icon"></button> </div> `; @@ -144,7 +144,7 @@ class GenericConfigView { this._div.id = 'config-item-' + this.item.getId(); this._div.appendChild(this._pre); - addEditButtons(this._div, this.item); + addEditButtons(this._div, this.item); this.item.kind.appendView(this._div); @@ -246,7 +246,7 @@ class LogConfigView { this._div.appendChild(this._pre); this._div.appendChild(this._table); - addEditButtons(this._div, this.item); + addEditButtons(this._div, this.item); this.item.kind.appendView(this._div); @@ -351,7 +351,7 @@ class NodeBrokerConfigView { this._div.id = 'config-item-' + this.item.getId(); this._div.appendChild(this._pre); - addEditButtons(this._div, this.item); + addEditButtons(this._div, this.item); this.item.kind.appendView(this._div); @@ -422,7 +422,7 @@ class ImmediateControlsConfigView { this._div.id = 'config-item-' + this.item.getId(); this._div.appendChild(this._pre); - addEditButtons(this._div, this.item); + addEditButtons(this._div, this.item); this.item.kind.appendView(this._div); @@ -873,7 +873,7 @@ class LogConfigEditForm extends CommonEditForm { super(kind, item); var form = $(this._form); - var syslog; + var syslog; var defLevel; var defSampling; var defRate; @@ -882,7 +882,7 @@ class LogConfigEditForm extends CommonEditForm { if (item && item.getConfig() && item.getConfig().LogConfig) { cfg = item.getConfig().LogConfig; - syslog = cfg.SysLog; + syslog = cfg.SysLog; defLevel = cfg.DefaultLevel; defSampling = cfg.DefaultSamplingLevel; defRate = cfg.DefaultSamplingRate; @@ -890,16 +890,16 @@ class LogConfigEditForm extends CommonEditForm { var configFormHTML = ` <div style="padding-left: 10px;"> - <div class="form-group row"> - <label class="col-form-label form-control-sm col-sm-1">Syslog:</label> - <div class="col-sm-auto"> - <select class="cfg-form syslog form-control-sm"> - <option value="-1" ${syslog === undefined ? "selected" : ""}>Default</option> - <option value="1" ${syslog !== undefined && syslog == true ? "selected" : ""}>true</option> - <option value="0" ${syslog !== undefined && syslog == false ? "selected" : ""}>false</option> - </select> - </div> - </div> + <div class="form-group row"> + <label class="col-form-label form-control-sm col-sm-1">Syslog:</label> + <div class="col-sm-auto"> + <select class="cfg-form syslog form-control-sm"> + <option value="-1" ${syslog === undefined ? "selected" : ""}>Default</option> + <option value="1" ${syslog !== undefined && syslog == true ? "selected" : ""}>true</option> + <option value="0" ${syslog !== undefined && syslog == false ? "selected" : ""}>false</option> + </select> + </div> + </div> <div class="form-group def-log-settings"> <div class="row"> <label class="col-form-label form-control-sm col-sm-auto">Default log settings:</label> @@ -1015,15 +1015,15 @@ class LogConfigEditForm extends CommonEditForm { _prepareData() { var data = super._prepareData(); - var syslog = +$(this._form).find(".syslog").children(":selected").val(); + var syslog = +$(this._form).find(".syslog").children(":selected").val(); var defSettings = $(this._form).find("div.def-log-settings"); var defLevel = +defSettings.find(".def-log-level").children(":selected").val(); var defSampling = +defSettings.find(".def-log-sampling").children(":selected").val(); var defRate = defSettings.find(".def-log-rate").val(); var config = { LogConfig: {} }; - if (syslog != -1) - config.LogConfig.SysLog = syslog ? true : false; + if (syslog != -1) + config.LogConfig.SysLog = syslog ? true : false; if (defLevel != -1) config.LogConfig.DefaultLevel = defLevel; if (defSampling != -1) @@ -1080,19 +1080,19 @@ class CmsConfigEditForm extends CommonEditForm { var monFaultyPrepTimeout; var logTTL; - // sentinel - var sentinelEnabled; - var sentinelDryRun; - var sentinelUpdateConfigInterval; - var sentinelRetryUpdateConfig; - var sentinelUpdateStateInterval; - var sentinelUpdateStateTimeout; - var sentinelChangeStatusRetries; - var sentinelRetryChangeStatus; - var sentinelDefaultStateLimit; - var sentinelStateLimits = new Map(); - var sentinelDefaultStateLimits = new Map(); - + // sentinel + var sentinelEnabled; + var sentinelDryRun; + var sentinelUpdateConfigInterval; + var sentinelRetryUpdateConfig; + var sentinelUpdateStateInterval; + var sentinelUpdateStateTimeout; + var sentinelChangeStatusRetries; + var sentinelRetryChangeStatus; + var sentinelDefaultStateLimit; + var sentinelStateLimits = new Map(); + var sentinelDefaultStateLimits = new Map(); + if (item && item.getConfig() && item.getConfig().CmsConfig) { var cfg = item.getConfig().CmsConfig; @@ -1113,42 +1113,42 @@ class CmsConfigEditForm extends CommonEditForm { if (cfg.LogConfig && cfg.LogConfig.TTL) logTTL = cfg.LogConfig.TTL / 1000000 / 3600 / 24; - - // sentinel - if (cfg.SentinelConfig) { - sentinelEnabled = cfg.SentinelConfig.Enable; - sentinelDryRun = cfg.SentinelConfig.DryRun; - if (cfg.SentinelConfig.UpdateConfigInterval) - sentinelUpdateConfigInterval = cfg.SentinelConfig.UpdateConfigInterval / 1000000; - if (cfg.SentinelConfig.RetryUpdateConfig) - sentinelRetryUpdateConfig = cfg.SentinelConfig.RetryUpdateConfig / 1000000; - if (cfg.SentinelConfig.UpdateStateInterval) - sentinelUpdateStateInterval = cfg.SentinelConfig.UpdateStateInterval / 1000000; - if (cfg.SentinelConfig.UpdateStateTimeout) - sentinelUpdateStateTimeout = cfg.SentinelConfig.UpdateStateTimeout / 1000000; - if (cfg.SentinelConfig.ChangeStatusRetries) - sentinelChangeStatusRetries = cfg.SentinelConfig.ChangeStatusRetries; - if (cfg.SentinelConfig.RetryChangeStatus) - sentinelRetryChangeStatus = cfg.SentinelConfig.RetryChangeStatus / 1000000; - if (cfg.SentinelConfig.DefaultStateLimit) - sentinelDefaultStateLimit = cfg.SentinelConfig.DefaultStateLimit; - if (cfg.SentinelConfig.StateLimits) { - for (var c of cfg.SentinelConfig.StateLimits) { - sentinelStateLimits.set(c.State, c.Limit); - } - } - } - } - - for (var e of cmsEnums['PDiskStates'].keys()) { - var name = cmsEnums.get('PDiskStates', e); - if (name.match(/Error$/)) { - sentinelDefaultStateLimits.set(e, 1); - } else if (e == 255) { - sentinelDefaultStateLimits.set(e, 0); - } else if (e > 250) { - sentinelDefaultStateLimits.set(e, 60); - } + + // sentinel + if (cfg.SentinelConfig) { + sentinelEnabled = cfg.SentinelConfig.Enable; + sentinelDryRun = cfg.SentinelConfig.DryRun; + if (cfg.SentinelConfig.UpdateConfigInterval) + sentinelUpdateConfigInterval = cfg.SentinelConfig.UpdateConfigInterval / 1000000; + if (cfg.SentinelConfig.RetryUpdateConfig) + sentinelRetryUpdateConfig = cfg.SentinelConfig.RetryUpdateConfig / 1000000; + if (cfg.SentinelConfig.UpdateStateInterval) + sentinelUpdateStateInterval = cfg.SentinelConfig.UpdateStateInterval / 1000000; + if (cfg.SentinelConfig.UpdateStateTimeout) + sentinelUpdateStateTimeout = cfg.SentinelConfig.UpdateStateTimeout / 1000000; + if (cfg.SentinelConfig.ChangeStatusRetries) + sentinelChangeStatusRetries = cfg.SentinelConfig.ChangeStatusRetries; + if (cfg.SentinelConfig.RetryChangeStatus) + sentinelRetryChangeStatus = cfg.SentinelConfig.RetryChangeStatus / 1000000; + if (cfg.SentinelConfig.DefaultStateLimit) + sentinelDefaultStateLimit = cfg.SentinelConfig.DefaultStateLimit; + if (cfg.SentinelConfig.StateLimits) { + for (var c of cfg.SentinelConfig.StateLimits) { + sentinelStateLimits.set(c.State, c.Limit); + } + } + } + } + + for (var e of cmsEnums['PDiskStates'].keys()) { + var name = cmsEnums.get('PDiskStates', e); + if (name.match(/Error$/)) { + sentinelDefaultStateLimits.set(e, 1); + } else if (e == 255) { + sentinelDefaultStateLimits.set(e, 0); + } else if (e > 250) { + sentinelDefaultStateLimits.set(e, 60); + } } var configFormHTML = ` @@ -1195,84 +1195,84 @@ class CmsConfigEditForm extends CommonEditForm { </div> <div class="form-group"> <div class="row"> - <label class="col-form-label form-control-sm col-sm-auto">Sentinel (self heal) parameters</label> + <label class="col-form-label form-control-sm col-sm-auto">Sentinel (self heal) parameters</label> + </div> + <div class="row"> + <div class="col-3"> + <label class="col-form-label form-control-sm">Status</label> + </div> + <div class="col-auto"> + <select class="form-control-sm sentinel-enabled"> + <option value="0" ${sentinelEnabled ? "" : "selected"}>Disabled</option> + <option value="1" ${sentinelEnabled ? "selected" : ""}>Enabled</option> + </select> + </div> </div> + <div class="w-100 vs-4px"></div> <div class="row"> <div class="col-3"> - <label class="col-form-label form-control-sm">Status</label> + <label class="col-form-label form-control-sm">Dry run</label> </div> <div class="col-auto"> - <select class="form-control-sm sentinel-enabled"> - <option value="0" ${sentinelEnabled ? "" : "selected"}>Disabled</option> - <option value="1" ${sentinelEnabled ? "selected" : ""}>Enabled</option> - </select> - </div> - </div> - <div class="w-100 vs-4px"></div> - <div class="row"> - <div class="col-3"> - <label class="col-form-label form-control-sm">Dry run</label> - </div> - <div class="col-auto"> - <select class="form-control-sm sentinel-dry-run"> - <option value="0" ${sentinelDryRun ? "" : "selected"}>Disable</option> - <option value="1" ${sentinelDryRun ? "selected" : ""}>Enable</option> - </select> - </div> - </div> - </div> - <div class="form-group"> - <div class="row"> - ${this._makeNumericInput("Config update interval (sec.)", "sentinel-update-config-interval", - "Default is 3600 sec.", sentinelUpdateConfigInterval, - "Interval used to update available (known by BSC) PDisks list")} - <div class="w-100 vs-4px"></div> - ${this._makeNumericInput("Retry interval (sec.)", "sentinel-retry-update-config", - "Default is 60 sec.", sentinelRetryUpdateConfig, - "Used if BSC is temporarily unavailable.")} - </div> - </div> - <div class="form-group"> - <div class="row"> - ${this._makeNumericInput("State update interval (sec.)", "sentinel-update-state-interval", - "Default is 60 sec.", sentinelUpdateStateInterval, - "Interval used to update states of PDisks")} - <div class="w-100 vs-4px"></div> - ${this._makeNumericInput("Timeout (sec.)", "sentinel-update-state-timeout", - "Default is 45 sec.", sentinelUpdateStateTimeout, - "Maximum allowed time for PDisks states collection")} - </div> - </div> - <div class="form-group"> - <div class="row"> - ${this._makeNumericInput("Change status retries", "sentinel-change-status-retries", - "Default is 5", sentinelChangeStatusRetries, - "Number of retries if status of PDisk cannot be changed due to BSC unavailability")} - <div class="w-100 vs-4px"></div> - ${this._makeNumericInput("Change status retry interval (sec.)", "sentinel-change-status-retry-interval", - "Default is 10 sec.", sentinelRetryChangeStatus, - "Used if BSC is temporarily unavailable.")} - </div> - </div> - <div class="form-group"> - <div class="row"> - ${this._makeNumericInput("Default state limit", "sentinel-default-state-limit", - "Default is 5", sentinelDefaultStateLimit, - "Number of 'state update' cycles before changing status")} - ${this._makeSentinelStateLimits(sentinelStateLimits, sentinelDefaultStateLimits)} - </div> - </div> - <div class="form-group"> - <div class="row"> - <label class="col-form-label form-control-sm col-sm-auto">Persistent log settings</label> - </div> - <div class="row"> - ${this._makeLogLevelSelects()} - <div class="w-100 vs-4px"></div> - ${this._makeNumericInput("Time to live (days)", "log-ttl", - "Default is 14 days", logTTL)} - </div> - </div> + <select class="form-control-sm sentinel-dry-run"> + <option value="0" ${sentinelDryRun ? "" : "selected"}>Disable</option> + <option value="1" ${sentinelDryRun ? "selected" : ""}>Enable</option> + </select> + </div> + </div> + </div> + <div class="form-group"> + <div class="row"> + ${this._makeNumericInput("Config update interval (sec.)", "sentinel-update-config-interval", + "Default is 3600 sec.", sentinelUpdateConfigInterval, + "Interval used to update available (known by BSC) PDisks list")} + <div class="w-100 vs-4px"></div> + ${this._makeNumericInput("Retry interval (sec.)", "sentinel-retry-update-config", + "Default is 60 sec.", sentinelRetryUpdateConfig, + "Used if BSC is temporarily unavailable.")} + </div> + </div> + <div class="form-group"> + <div class="row"> + ${this._makeNumericInput("State update interval (sec.)", "sentinel-update-state-interval", + "Default is 60 sec.", sentinelUpdateStateInterval, + "Interval used to update states of PDisks")} + <div class="w-100 vs-4px"></div> + ${this._makeNumericInput("Timeout (sec.)", "sentinel-update-state-timeout", + "Default is 45 sec.", sentinelUpdateStateTimeout, + "Maximum allowed time for PDisks states collection")} + </div> + </div> + <div class="form-group"> + <div class="row"> + ${this._makeNumericInput("Change status retries", "sentinel-change-status-retries", + "Default is 5", sentinelChangeStatusRetries, + "Number of retries if status of PDisk cannot be changed due to BSC unavailability")} + <div class="w-100 vs-4px"></div> + ${this._makeNumericInput("Change status retry interval (sec.)", "sentinel-change-status-retry-interval", + "Default is 10 sec.", sentinelRetryChangeStatus, + "Used if BSC is temporarily unavailable.")} + </div> + </div> + <div class="form-group"> + <div class="row"> + ${this._makeNumericInput("Default state limit", "sentinel-default-state-limit", + "Default is 5", sentinelDefaultStateLimit, + "Number of 'state update' cycles before changing status")} + ${this._makeSentinelStateLimits(sentinelStateLimits, sentinelDefaultStateLimits)} + </div> + </div> + <div class="form-group"> + <div class="row"> + <label class="col-form-label form-control-sm col-sm-auto">Persistent log settings</label> + </div> + <div class="row"> + ${this._makeLogLevelSelects()} + <div class="w-100 vs-4px"></div> + ${this._makeNumericInput("Time to live (days)", "log-ttl", + "Default is 14 days", logTTL)} + </div> + </div> </div> `; @@ -1346,32 +1346,32 @@ class CmsConfigEditForm extends CommonEditForm { return res; } - _makeSentinelStateLimits(stateLimits, defaultStateLimits) { - var res = ''; - - for (var e of cmsEnums['PDiskStates'].keys()) { - var name = cmsEnums.get('PDiskStates', e); - var placeholder = "Use default if empty"; - if (stateLimits.size == 0 && defaultStateLimits.has(e)) { - placeholder = "Default is " + defaultStateLimits.get(e) + " cycles"; - } - - var val = stateLimits.has(e) ? stateLimits.get(e) : ''; - res += ` - <div class="w-100 vs-4px"></div> - <div class="col-3"> - <label class="col-form-label form-control-sm"><i>${name}</i></label> - </div> - <div class="col-auto"> - <input type="number" class="form-control-sm cfg-form sentinel-state-limit" placeholder="${placeholder}" value="${val}" data-recordtype="${e}"/> - </div> - <div class="help-icon" data-toggle="tooltip" title="Number of 'state update' cycles before changing status"></div> - `; - } - - return res; - } - + _makeSentinelStateLimits(stateLimits, defaultStateLimits) { + var res = ''; + + for (var e of cmsEnums['PDiskStates'].keys()) { + var name = cmsEnums.get('PDiskStates', e); + var placeholder = "Use default if empty"; + if (stateLimits.size == 0 && defaultStateLimits.has(e)) { + placeholder = "Default is " + defaultStateLimits.get(e) + " cycles"; + } + + var val = stateLimits.has(e) ? stateLimits.get(e) : ''; + res += ` + <div class="w-100 vs-4px"></div> + <div class="col-3"> + <label class="col-form-label form-control-sm"><i>${name}</i></label> + </div> + <div class="col-auto"> + <input type="number" class="form-control-sm cfg-form sentinel-state-limit" placeholder="${placeholder}" value="${val}" data-recordtype="${e}"/> + </div> + <div class="help-icon" data-toggle="tooltip" title="Number of 'state update' cycles before changing status"></div> + `; + } + + return res; + } + _prepareData() { var data = super._prepareData(); var cfg = {}; @@ -1404,28 +1404,28 @@ class CmsConfigEditForm extends CommonEditForm { } }); - // sentinel - this._setBoolSelectValue(cfg, 'sentinel-enabled', ['SentinelConfig', 'Enable']); - this._setBoolSelectValue(cfg, 'sentinel-dry-run', ['SentinelConfig', 'DryRun']); - this._setTimeSecValue(cfg, 'sentinel-update-config-interval', ['SentinelConfig', 'UpdateConfigInterval']); - this._setTimeSecValue(cfg, 'sentinel-retry-update-config', ['SentinelConfig', 'RetryUpdateConfig']); - this._setTimeSecValue(cfg, 'sentinel-update-state-interval', ['SentinelConfig', 'UpdateStateInterval']); - this._setTimeSecValue(cfg, 'sentinel-update-state-timeout', ['SentinelConfig', 'UpdateStateTimeout']); - this._setNumValue(cfg, 'sentinel-change-status-retries', ['SentinelConfig', 'ChangeStatusRetries']); - this._setTimeSecValue(cfg, 'sentinel-change-status-retry-interval', ['SentinelConfig', 'RetryChangeStatus']); - this._setNumValue(cfg, 'sentinel-default-state-limit', ['SentinelConfig', 'DefaultStateLimit']); - $(this._form).find('input.sentinel-state-limit').each(function() { - var val = $(this).val(); - if (val) { - if (cfg.SentinelConfig.StateLimits === undefined) - cfg.SentinelConfig.StateLimits = []; - cfg.SentinelConfig.StateLimits.push({ - State: +this.dataset.recordtype, - Limit: +val, - }); - } - }); - + // sentinel + this._setBoolSelectValue(cfg, 'sentinel-enabled', ['SentinelConfig', 'Enable']); + this._setBoolSelectValue(cfg, 'sentinel-dry-run', ['SentinelConfig', 'DryRun']); + this._setTimeSecValue(cfg, 'sentinel-update-config-interval', ['SentinelConfig', 'UpdateConfigInterval']); + this._setTimeSecValue(cfg, 'sentinel-retry-update-config', ['SentinelConfig', 'RetryUpdateConfig']); + this._setTimeSecValue(cfg, 'sentinel-update-state-interval', ['SentinelConfig', 'UpdateStateInterval']); + this._setTimeSecValue(cfg, 'sentinel-update-state-timeout', ['SentinelConfig', 'UpdateStateTimeout']); + this._setNumValue(cfg, 'sentinel-change-status-retries', ['SentinelConfig', 'ChangeStatusRetries']); + this._setTimeSecValue(cfg, 'sentinel-change-status-retry-interval', ['SentinelConfig', 'RetryChangeStatus']); + this._setNumValue(cfg, 'sentinel-default-state-limit', ['SentinelConfig', 'DefaultStateLimit']); + $(this._form).find('input.sentinel-state-limit').each(function() { + var val = $(this).val(); + if (val) { + if (cfg.SentinelConfig.StateLimits === undefined) + cfg.SentinelConfig.StateLimits = []; + cfg.SentinelConfig.StateLimits.push({ + State: +this.dataset.recordtype, + Limit: +val, + }); + } + }); + data.Config = { CmsConfig: cfg }; return data; diff --git a/ydb/core/cms/ui/configs.js b/ydb/core/cms/ui/configs.js index 5cfd08d3995..f3b807e451b 100644 --- a/ydb/core/cms/ui/configs.js +++ b/ydb/core/cms/ui/configs.js @@ -328,6 +328,6 @@ function initConfigsTab() { .add('ServiceKikimr', 'NKikimrServices::EServiceKikimr') .add('CmsLogConfigLevel', 'NKikimrCms::TCmsConfig::TLogConfig::ELevel') .add('CmsLogRecordType', 'NKikimrCms::TLogRecordData::EType') - .add('PDiskStates', 'NCms::EPDiskState') + .add('PDiskStates', 'NCms::EPDiskState') .done(loadTypes); } diff --git a/ydb/core/cms/ut/ya.make b/ydb/core/cms/ut/ya.make index 6bea8c38b84..91fb1cd823e 100644 --- a/ydb/core/cms/ut/ya.make +++ b/ydb/core/cms/ut/ya.make @@ -27,7 +27,7 @@ SRCS( cms_tenants_ut.cpp cms_ut_common.cpp cms_ut_common.h - downtime_ut.cpp + downtime_ut.cpp ut_helpers.cpp ) diff --git a/ydb/core/cms/ut_helpers.h b/ydb/core/cms/ut_helpers.h index 0a6f758bc7b..ba2a0fa0505 100644 --- a/ydb/core/cms/ut_helpers.h +++ b/ydb/core/cms/ut_helpers.h @@ -369,16 +369,16 @@ private: NKikimrBlobStorage::EDriveStatus Status; }; -inline NKikimrWhiteboard::TSystemStateInfo MakeSystemStateInfo(const TString& version, const TVector<TString>& roles = {}) { - NKikimrWhiteboard::TSystemStateInfo result; - - result.SetVersion(version); - for (const auto& role : roles) { - result.AddRoles(role); - } - - return result; -} - +inline NKikimrWhiteboard::TSystemStateInfo MakeSystemStateInfo(const TString& version, const TVector<TString>& roles = {}) { + NKikimrWhiteboard::TSystemStateInfo result; + + result.SetVersion(version); + for (const auto& role : roles) { + result.AddRoles(role); + } + + return result; +} + } // NCmsTest } // NKikimr diff --git a/ydb/core/cms/ut_sentinel/ya.make b/ydb/core/cms/ut_sentinel/ya.make index 3389b721450..773d06889ad 100644 --- a/ydb/core/cms/ut_sentinel/ya.make +++ b/ydb/core/cms/ut_sentinel/ya.make @@ -1,26 +1,26 @@ UNITTEST_FOR(ydb/core/cms) - -OWNER( - ilnaz - g:kikimr -) - -SIZE(MEDIUM) -TIMEOUT(600) - -PEERDIR( +OWNER( + ilnaz + g:kikimr +) + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/getopt library/cpp/svnversion ydb/core/testlib -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - cms_ut_common.cpp - cms_ut_common.h - sentinel_ut.cpp -) - -END() +SRCS( + cms_ut_common.cpp + cms_ut_common.h + sentinel_ut.cpp +) + +END() diff --git a/ydb/core/cms/walle.h b/ydb/core/cms/walle.h index 24271017344..9a08bb45758 100644 --- a/ydb/core/cms/walle.h +++ b/ydb/core/cms/walle.h @@ -1,6 +1,6 @@ #pragma once -#include "audit_log.h" +#include "audit_log.h" #include "cms_state.h" #include "base_handler.h" #include "cms.h" @@ -31,9 +31,9 @@ public: } }; -inline void WalleAuditLog(const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { - AuditLog("Wall-E adapter", request, response, ctx); -} +inline void WalleAuditLog(const IEventBase* request, const IEventBase* response, const TActorContext& ctx) { + AuditLog("Wall-E adapter", request, response, ctx); +} } // NCms } // NKikimr diff --git a/ydb/core/cms/walle_api_handler.cpp b/ydb/core/cms/walle_api_handler.cpp index 5ab96db90cd..7b62f3275a9 100644 --- a/ydb/core/cms/walle_api_handler.cpp +++ b/ydb/core/cms/walle_api_handler.cpp @@ -44,7 +44,7 @@ public: else if (msg->Request.GetMethod() == HTTP_METHOD_POST) ProcessCreateTasksRequest(msg->Request, ctx); else - ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); + ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); } else if (path.StartsWith("tasks/")) { TString id = path.substr(strlen("tasks/")); if (msg->Request.GetMethod() == HTTP_METHOD_GET) @@ -52,9 +52,9 @@ public: else if (msg->Request.GetMethod() == HTTP_METHOD_DELETE) ProcessRemoveTaskRequest(id, ctx); else - ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); + ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); } else { - ReplyWithError(NMonitoring::HTTPNOTFOUND, ctx); + ReplyWithError(NMonitoring::HTTPNOTFOUND, ctx); } } @@ -69,7 +69,7 @@ private: try { ReadJsonTree(http.GetPostContent(), &json, true); } catch (yexception e) { - ReplyWithError(TString("HTTP/1.1 400 Bad Request\r\n\r\nCan't parse provided JSON: ") + e.what(), ctx); + ReplyWithError(TString("HTTP/1.1 400 Bad Request\r\n\r\nCan't parse provided JSON: ") + e.what(), ctx); return; } @@ -89,7 +89,7 @@ private: if (params.contains("dry_run")) request->Record.SetDryRun(params.find("dry_run")->second == "true"); - SendToCms(request.Release(), ctx); + SendToCms(request.Release(), ctx); Become(&TThis::StateCreateTask, ctx, TDuration::Seconds(60), new TEvents::TEvWakeup()); } @@ -97,8 +97,8 @@ private: { switch (ev->GetTypeRewrite()) { HFunc(TEvCms::TEvWalleCreateTaskResponse, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); CFunc(TEvents::TSystem::Wakeup, Timeout); default: LOG_DEBUG(ctx, NKikimrServices::CMS, "TWalleApiHandler::StateCreateTask ignored event type: %" PRIx32 " event: %s", @@ -106,7 +106,7 @@ private: } } - void Reply(const TWalleCreateTaskResponse &resp, const TActorContext &ctx) + void Reply(const TWalleCreateTaskResponse &resp, const TActorContext &ctx) { TString status; if (!CheckStatus(resp.GetStatus(), status, false, ctx)) @@ -121,12 +121,12 @@ private: map.InsertValue("status", status); map.InsertValue("message", resp.GetStatus().GetReason()); - Reply(WriteJson(map), ctx); + Reply(WriteJson(map), ctx); } void Handle(TEvCms::TEvWalleCreateTaskResponse::TPtr &ev, const TActorContext &ctx) { - Reply(ev->Get()->Record, ctx); + Reply(ev->Get()->Record, ctx); } ////////////////////////////////////////////// @@ -136,7 +136,7 @@ private: { TAutoPtr<TEvCms::TEvWalleListTasksRequest> request = new TEvCms::TEvWalleListTasksRequest; - SendToCms(request.Release(), ctx); + SendToCms(request.Release(), ctx); Become(&TThis::StateListTasks, ctx, TDuration::Seconds(60), new TEvents::TEvWakeup()); } @@ -144,8 +144,8 @@ private: { switch (ev->GetTypeRewrite()) { HFunc(TEvCms::TEvWalleListTasksResponse, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); CFunc(TEvents::TSystem::Wakeup, Timeout); default: LOG_DEBUG(ctx, NKikimrServices::CMS, "TWalleApiHandler::StateListTasks ignored event type: %" PRIx32 " event: %s", @@ -153,7 +153,7 @@ private: } } - void Reply(const TWalleListTasksResponse &resp, const TActorContext &ctx) + void Reply(const TWalleListTasksResponse &resp, const TActorContext &ctx) { NJson::TJsonValue tasks(NJson::JSON_ARRAY); for (auto &task : resp.GetTasks()) { @@ -171,12 +171,12 @@ private: NJson::TJsonValue res; res.InsertValue("result", tasks); - Reply(WriteJson(res), ctx); + Reply(WriteJson(res), ctx); } void Handle(TEvCms::TEvWalleListTasksResponse::TPtr &ev, const TActorContext &ctx) { - Reply(ev->Get()->Record, ctx); + Reply(ev->Get()->Record, ctx); } ////////////////////////////////////////////// @@ -187,7 +187,7 @@ private: TAutoPtr<TEvCms::TEvWalleCheckTaskRequest> request = new TEvCms::TEvWalleCheckTaskRequest; request->Record.SetTaskId(id); - SendToCms(request.Release(), ctx); + SendToCms(request.Release(), ctx); Become(&TThis::StateCheckTask, ctx, TDuration::Seconds(60), new TEvents::TEvWakeup()); } @@ -195,8 +195,8 @@ private: { switch (ev->GetTypeRewrite()) { HFunc(TEvCms::TEvWalleCheckTaskResponse, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); CFunc(TEvents::TSystem::Wakeup, Timeout); default: LOG_DEBUG(ctx, NKikimrServices::CMS, "TWalleApiHandler::StateCheckTask ignored event type: %" PRIx32 " event: %s", @@ -204,7 +204,7 @@ private: } } - void Reply(const TWalleCheckTaskResponse &resp, const TActorContext &ctx) + void Reply(const TWalleCheckTaskResponse &resp, const TActorContext &ctx) { TString status; if (!CheckStatus(resp.GetStatus(), status, true, ctx)) @@ -219,12 +219,12 @@ private: map.InsertValue("status", status); map.InsertValue("message", resp.GetStatus().GetReason()); - Reply(WriteJson(map), ctx); + Reply(WriteJson(map), ctx); } void Handle(TEvCms::TEvWalleCheckTaskResponse::TPtr &ev, const TActorContext &ctx) { - Reply(ev->Get()->Record, ctx); + Reply(ev->Get()->Record, ctx); } ////////////////////////////////////////////// @@ -235,7 +235,7 @@ private: TAutoPtr<TEvCms::TEvWalleRemoveTaskRequest> request = new TEvCms::TEvWalleRemoveTaskRequest; request->Record.SetTaskId(id); - SendToCms(request.Release(), ctx); + SendToCms(request.Release(), ctx); Become(&TThis::StateRemoveTask, ctx, TDuration::Seconds(60), new TEvents::TEvWakeup()); } @@ -243,8 +243,8 @@ private: { switch (ev->GetTypeRewrite()) { HFunc(TEvCms::TEvWalleRemoveTaskResponse, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); CFunc(TEvents::TSystem::Wakeup, Timeout); default: LOG_DEBUG(ctx, NKikimrServices::CMS, "TWalleApiHandler::StateRemoveTask ignored event type: %" PRIx32 " event: %s", @@ -252,18 +252,18 @@ private: } } - void Reply(const TWalleRemoveTaskResponse &resp, const TActorContext &ctx) + void Reply(const TWalleRemoveTaskResponse &resp, const TActorContext &ctx) { TString status; if (!CheckStatus(resp.GetStatus(), status, true, ctx)) return; - Reply("", ctx); + Reply("", ctx); } void Handle(TEvCms::TEvWalleRemoveTaskResponse::TPtr &ev, const TActorContext &ctx) { - Reply(ev->Get()->Record, ctx); + Reply(ev->Get()->Record, ctx); } ////////////////////////////////////////////// @@ -293,29 +293,29 @@ private: err = NMonitoring::HTTPNOTFOUND; else err = Sprintf("HTTP/1.1 400 Bad Request\r\n\r\n%s", status.GetReason().data()); - ReplyWithError(err, ctx); + ReplyWithError(err, ctx); return false; } auto err = Sprintf("HTTP/1.1 500 Internal Server Error\r\n\r\n%s", status.GetReason().data()); - ReplyWithError(err, ctx); + ReplyWithError(err, ctx); return false; } - void ReplyWithError(const TString &err, const TActorContext &ctx) + void ReplyWithError(const TString &err, const TActorContext &ctx) { - return ReplyAndDie(err, ctx); + return ReplyAndDie(err, ctx); } - void Reply(const TString &json, const TActorContext &ctx) + void Reply(const TString &json, const TActorContext &ctx) { - const TString header = json ? NMonitoring::HTTPOKJSON : NMonitoring::HTTPNOCONTENT; - return ReplyAndDie(header + json, ctx); - } - - void ReplyAndDie(const TString &data, const TActorContext &ctx) { - AuditLog("Wall-E handler", RequestEvent, data, ctx); - ctx.Send(RequestEvent->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + const TString header = json ? NMonitoring::HTTPOKJSON : NMonitoring::HTTPNOCONTENT; + return ReplyAndDie(header + json, ctx); + } + + void ReplyAndDie(const TString &data, const TActorContext &ctx) { + AuditLog("Wall-E handler", RequestEvent, data, ctx); + ctx.Send(RequestEvent->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } @@ -327,50 +327,50 @@ private: void Timeout(const TActorContext& ctx) { - ReplyWithError(TString("HTTP/1.1 408 Request Timeout\r\n\r\nCMS request timeout"), ctx); + ReplyWithError(TString("HTTP/1.1 408 Request Timeout\r\n\r\nCMS request timeout"), ctx); } - - void SendToCms(IEventBase* ev, const TActorContext &ctx) { - Y_VERIFY(!CmsPipe); - - ui32 domain = AppData(ctx)->DomainsInfo->Domains.begin()->first; - NTabletPipe::TClientConfig pipeConfig; + + void SendToCms(IEventBase* ev, const TActorContext &ctx) { + Y_VERIFY(!CmsPipe); + + ui32 domain = AppData(ctx)->DomainsInfo->Domains.begin()->first; + NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = { .MinRetryTime = TDuration::MilliSeconds(10), .MaxRetryTime = TDuration::Seconds(10), }; CmsPipe = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, MakeCmsID(domain), pipeConfig)); - NTabletPipe::SendData(ctx, CmsPipe, ev); - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { - TEvTabletPipe::TEvClientConnected *msg = ev->Get(); - - if (msg->ClientId != CmsPipe || msg->Status == NKikimrProto::OK) { - return; - } - - HandlePipeDestroyed(ctx); - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx) { - TEvTabletPipe::TEvClientDestroyed *msg = ev->Get(); - - if (msg->ClientId != CmsPipe) { - return; - } - - HandlePipeDestroyed(ctx); - } - - void HandlePipeDestroyed(const TActorContext &ctx) { - LOG_WARN(ctx, NKikimrServices::CMS, "TWalleApiHandler::HandlePipeDestroyed"); - - NTabletPipe::CloseAndForgetClient(SelfId(), CmsPipe); - Bootstrap(ctx); - } - + NTabletPipe::SendData(ctx, CmsPipe, ev); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { + TEvTabletPipe::TEvClientConnected *msg = ev->Get(); + + if (msg->ClientId != CmsPipe || msg->Status == NKikimrProto::OK) { + return; + } + + HandlePipeDestroyed(ctx); + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx) { + TEvTabletPipe::TEvClientDestroyed *msg = ev->Get(); + + if (msg->ClientId != CmsPipe) { + return; + } + + HandlePipeDestroyed(ctx); + } + + void HandlePipeDestroyed(const TActorContext &ctx) { + LOG_WARN(ctx, NKikimrServices::CMS, "TWalleApiHandler::HandlePipeDestroyed"); + + NTabletPipe::CloseAndForgetClient(SelfId(), CmsPipe); + Bootstrap(ctx); + } + NMon::TEvHttpInfo::TPtr RequestEvent; TActorId CmsPipe; }; diff --git a/ydb/core/cms/walle_check_task_adapter.cpp b/ydb/core/cms/walle_check_task_adapter.cpp index cf1df2a57bb..e5570c9af91 100644 --- a/ydb/core/cms/walle_check_task_adapter.cpp +++ b/ydb/core/cms/walle_check_task_adapter.cpp @@ -27,7 +27,7 @@ public: { TString id = RequestEvent->Get()->Record.GetTaskId(); - LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", RequestEvent->Get()->Record.ShortDebugString().data()); if (!State->WalleTasks.contains(id)) { @@ -93,7 +93,7 @@ private: void ReplyAndDie(TAutoPtr<TEvCms::TEvWalleCheckTaskResponse> resp, const TActorContext &ctx) { - WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); + WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); ctx.Send(RequestEvent->Sender, resp.Release()); Die(ctx); } diff --git a/ydb/core/cms/walle_create_task_adapter.cpp b/ydb/core/cms/walle_create_task_adapter.cpp index c6eda99e143..56989e2db03 100644 --- a/ydb/core/cms/walle_create_task_adapter.cpp +++ b/ydb/core/cms/walle_create_task_adapter.cpp @@ -26,7 +26,7 @@ public: { auto &rec = RequestEvent->Get()->Record; - LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", rec.ShortDebugString().data()); if (rec.GetAction() != "reboot" @@ -83,7 +83,7 @@ private: void ReplyAndDie(TAutoPtr<TEvCms::TEvWalleCreateTaskResponse> resp, const TActorContext &ctx) { - WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); + WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); ctx.Send(RequestEvent->Sender, resp.Release()); Die(ctx); } @@ -149,38 +149,38 @@ private: ReplyAndDie(resp.Release(), ctx); return; } else { - // We always use infinite duration. - // Wall-E MUST delete processed tasks. + // We always use infinite duration. + // Wall-E MUST delete processed tasks. if (task.GetAction() == "reboot") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "power-off") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "change-disk") { action.SetType(TAction::REPLACE_DEVICES); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "change-memory") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "profile") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "redeploy") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "repair-link") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "repair-bmc") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "repair-overheat") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else if (task.GetAction() == "repair-capping") { action.SetType(TAction::SHUTDOWN_HOST); - action.SetDuration(TDuration::Max().GetValue()); + action.SetDuration(TDuration::Max().GetValue()); } else Y_FAIL("Unknown action"); diff --git a/ydb/core/cms/walle_list_tasks_adapter.cpp b/ydb/core/cms/walle_list_tasks_adapter.cpp index 0aea772dee7..d256294cc18 100644 --- a/ydb/core/cms/walle_list_tasks_adapter.cpp +++ b/ydb/core/cms/walle_list_tasks_adapter.cpp @@ -23,11 +23,11 @@ public: void Bootstrap(const TActorContext &ctx) { - auto &rec = RequestEvent->Get()->Record; - - LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", - rec.ShortDebugString().data()); - + auto &rec = RequestEvent->Get()->Record; + + LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", + rec.ShortDebugString().data()); + TAutoPtr<TEvCms::TEvWalleListTasksResponse> response = new TEvCms::TEvWalleListTasksResponse; for (auto &entry : State->WalleTasks) { @@ -55,7 +55,7 @@ public: private: void ReplyAndDie(TAutoPtr<TEvCms::TEvWalleListTasksResponse> resp, const TActorContext &ctx) { - WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); + WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); ctx.Send(RequestEvent->Sender, resp.Release()); Die(ctx); } diff --git a/ydb/core/cms/walle_remove_task_adapter.cpp b/ydb/core/cms/walle_remove_task_adapter.cpp index 6991cd8aa95..899d4848250 100644 --- a/ydb/core/cms/walle_remove_task_adapter.cpp +++ b/ydb/core/cms/walle_remove_task_adapter.cpp @@ -27,7 +27,7 @@ public: TAutoPtr<TEvCms::TEvWalleRemoveTaskResponse> response = new TEvCms::TEvWalleRemoveTaskResponse; TString id = RequestEvent->Get()->Record.GetTaskId(); - LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", + LOG_INFO(ctx, NKikimrServices::CMS, "Processing Wall-E request: %s", RequestEvent->Get()->Record.ShortDebugString().data()); if (!State->WalleTasks.contains(id)) { @@ -64,7 +64,7 @@ private: void ReplyAndDie(TAutoPtr<TEvCms::TEvWalleRemoveTaskResponse> resp, const TActorContext &ctx) { - WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); + WalleAuditLog(RequestEvent->Get(), resp.Get(), ctx); ctx.Send(RequestEvent->Sender, resp.Release()); Die(ctx); } diff --git a/ydb/core/cms/ya.make b/ydb/core/cms/ya.make index 03aa62e19cf..b341c4a0d09 100644 --- a/ydb/core/cms/ya.make +++ b/ydb/core/cms/ya.make @@ -6,7 +6,7 @@ OWNER( ) SRCS( - audit_log.cpp + audit_log.cpp base_handler.h cluster_info.cpp cluster_info.h @@ -49,8 +49,8 @@ SRCS( log_formatter.h pdiskid.h scheme.h - sentinel.cpp - services.cpp + sentinel.cpp + services.cpp walle.h walle_api_handler.cpp walle_check_task_adapter.cpp @@ -105,8 +105,8 @@ PEERDIR( ydb/library/aclib ) -GENERATE_ENUM_SERIALIZATION(services.h) - +GENERATE_ENUM_SERIALIZATION(services.h) + END() RECURSE( diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 1405fc835ce..450d4135c6c 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -2,7 +2,7 @@ #include "cli_cmds.h" #include <ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h> - + #include <ydb/library/aclib/aclib.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> @@ -13,7 +13,7 @@ #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/string/split.h> #include <util/string/join.h> #include <util/string/printf.h> @@ -268,9 +268,9 @@ public: case NKikimrSchemeOp::EPathTypeSequence: type = "<sequence>"; break; - case NKikimrSchemeOp::EPathTypeReplication: - type = "<replication>"; - break; + case NKikimrSchemeOp::EPathTypeReplication: + type = "<replication>"; + break; case NKikimrSchemeOp::EPathTypePersQueueGroup: type = "<pq group>"; break; @@ -483,9 +483,9 @@ public: case NKikimrSchemeOp::EPathTypeSequence: type = "<sequence>"; break; - case NKikimrSchemeOp::EPathTypeReplication: - type = "<replication>"; - break; + case NKikimrSchemeOp::EPathTypeReplication: + type = "<replication>"; + break; default: type = "<unknown>"; break; @@ -1019,58 +1019,58 @@ public: } }; -class TClientCommandSchemaUserAttributeGet: public TClientCommand { -public: - TClientCommandSchemaUserAttributeGet() - : TClientCommand("get", {}, "Get user attributes") - {} - - virtual void Config(TConfig& config) override { - TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); - } - - TString Path; - - virtual void Parse(TConfig& config) override { - TClientCommand::Parse(config); - TString pathname = config.ParseResult->GetFreeArgs()[0]; - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Path = config.Path + '/' + pathname; - } else { - WarnProfilePathSet(); - Path = pathname; - } - } else { - Path = pathname; - } - } - +class TClientCommandSchemaUserAttributeGet: public TClientCommand { +public: + TClientCommandSchemaUserAttributeGet() + : TClientCommand("get", {}, "Get user attributes") + {} + + virtual void Config(TConfig& config) override { + TClientCommand::Config(config); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); + } + + TString Path; + + virtual void Parse(TConfig& config) override { + TClientCommand::Parse(config); + TString pathname = config.ParseResult->GetFreeArgs()[0]; + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Path = config.Path + '/' + pathname; + } else { + WarnProfilePathSet(); + Path = pathname; + } + } else { + Path = pathname; + } + } + static void Print(const google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TUserAttribute>& items) { - for (const auto& item : items) { - Cout << item.GetKey() << ": " << item.GetValue() << Endl; - } - } - - virtual int Run(TConfig& config) override { - TAutoPtr<NMsgBusProxy::TBusSchemeDescribe> request(new NMsgBusProxy::TBusSchemeDescribe()); - NKikimrClient::TSchemeDescribe& record(request->Record); - - record.SetPath(Path); - - return MessageBusCall<NMsgBusProxy::TBusSchemeDescribe, NMsgBusProxy::TBusResponse>(config, request, - [](const NMsgBusProxy::TBusResponse& response) -> int { - Print(response.Record.GetPathDescription().GetUserAttributes()); - return 0; - } - ); - } -}; - + for (const auto& item : items) { + Cout << item.GetKey() << ": " << item.GetValue() << Endl; + } + } + + virtual int Run(TConfig& config) override { + TAutoPtr<NMsgBusProxy::TBusSchemeDescribe> request(new NMsgBusProxy::TBusSchemeDescribe()); + NKikimrClient::TSchemeDescribe& record(request->Record); + + record.SetPath(Path); + + return MessageBusCall<NMsgBusProxy::TBusSchemeDescribe, NMsgBusProxy::TBusResponse>(config, request, + [](const NMsgBusProxy::TBusResponse& response) -> int { + Print(response.Record.GetPathDescription().GetUserAttributes()); + return 0; + } + ); + } +}; + std::pair<TString, TString> SplitPath(const TClientCommand::TConfig& config, const TString& pathname) { std::pair<TString, TString> result; @@ -1093,93 +1093,93 @@ std::pair<TString, TString> SplitPath(const TClientCommand::TConfig& config, con return result; } -class TClientCommandSchemaUserAttributeSet: public TClientCommand { +class TClientCommandSchemaUserAttributeSet: public TClientCommand { using TUserAttributesLimits = NSchemeShard::TUserAttributesLimits; - -public: - TClientCommandSchemaUserAttributeSet() - : TClientCommand("set", {}, "Set user attribute(s)") - {} - - virtual void Config(TConfig& config) override { - TClientCommand::Config(config); - config.SetFreeArgsMin(2); - SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); - SetFreeArgTitle(1, "<ATTRIBUTE>", "NAME=VALUE"); - } - - TString Base; - TString Name; - THashMap<TString, TString> Attributes; - - virtual void Parse(TConfig& config) override { - TClientCommand::Parse(config); - + +public: + TClientCommandSchemaUserAttributeSet() + : TClientCommand("set", {}, "Set user attribute(s)") + {} + + virtual void Config(TConfig& config) override { + TClientCommand::Config(config); + config.SetFreeArgsMin(2); + SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); + SetFreeArgTitle(1, "<ATTRIBUTE>", "NAME=VALUE"); + } + + TString Base; + TString Name; + THashMap<TString, TString> Attributes; + + virtual void Parse(TConfig& config) override { + TClientCommand::Parse(config); + std::tie(Base, Name) = SplitPath(config, config.ParseResult->GetFreeArgs()[0]); - - for (ui32 i = 1; i < config.ParseResult->GetFreeArgCount(); ++i) { - TString attr = config.ParseResult->GetFreeArgs()[i]; - TVector<TString> items = StringSplitter(attr).Split('=').ToList<TString>(); - if (items.size() != 2) { - Cerr << "Bad format in attribute '" + attr + "'"; - exit(1); - } - - const TString& key = items.at(0); + + for (ui32 i = 1; i < config.ParseResult->GetFreeArgCount(); ++i) { + TString attr = config.ParseResult->GetFreeArgs()[i]; + TVector<TString> items = StringSplitter(attr).Split('=').ToList<TString>(); + if (items.size() != 2) { + Cerr << "Bad format in attribute '" + attr + "'"; + exit(1); + } + + const TString& key = items.at(0); if (key.size() > TUserAttributesLimits::MaxNameLen) { - Cerr << "Key '" << key << "' too long" + Cerr << "Key '" << key << "' too long" << ", max: " << TUserAttributesLimits::MaxNameLen - << ", actual: " << key.size() << Endl; - exit(1); - } - - const TString& value = items.at(1); + << ", actual: " << key.size() << Endl; + exit(1); + } + + const TString& value = items.at(1); if (value.size() > TUserAttributesLimits::MaxValueLen) { - Cerr << "Value '" << value << "' too long" + Cerr << "Value '" << value << "' too long" << ", max: " << TUserAttributesLimits::MaxValueLen - << ", actual: " << value.size() << Endl; - exit(1); - } - - if (Attributes.contains(key)) { - Cerr << "Duplicate value for key: " << key << Endl; - } - - Attributes[key] = value; - } - } - - virtual int Run(TConfig& config) override { - TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); - NKikimrClient::TSchemeOperation& record(request->Record); - - auto& modifyScheme = *record.MutableTransaction()->MutableModifyScheme(); + << ", actual: " << value.size() << Endl; + exit(1); + } + + if (Attributes.contains(key)) { + Cerr << "Duplicate value for key: " << key << Endl; + } + + Attributes[key] = value; + } + } + + virtual int Run(TConfig& config) override { + TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); + NKikimrClient::TSchemeOperation& record(request->Record); + + auto& modifyScheme = *record.MutableTransaction()->MutableModifyScheme(); modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes); - modifyScheme.SetWorkingDir(Base); - - auto& alter = *modifyScheme.MutableAlterUserAttributes(); - alter.SetPathName(Name); - auto& attributes = *alter.MutableUserAttributes(); - attributes.Reserve(Attributes.size()); - for (const auto& kv : Attributes) { - auto& attribute = *attributes.Add(); - attribute.SetKey(kv.first); - attribute.SetValue(kv.second); - } - - return MessageBusCall<NMsgBusProxy::TBusSchemeOperation, NMsgBusProxy::TBusResponse>(config, request, - [](const NMsgBusProxy::TBusResponse& response) -> int { - if (response.Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - Cerr << ToCString(static_cast<NMsgBusProxy::EResponseStatus>(response.Record.GetStatus())) << " " << response.Record.GetErrorReason() << Endl; - return 1; - } - return 0; - } - ); - } -}; - + modifyScheme.SetWorkingDir(Base); + + auto& alter = *modifyScheme.MutableAlterUserAttributes(); + alter.SetPathName(Name); + auto& attributes = *alter.MutableUserAttributes(); + attributes.Reserve(Attributes.size()); + for (const auto& kv : Attributes) { + auto& attribute = *attributes.Add(); + attribute.SetKey(kv.first); + attribute.SetValue(kv.second); + } + + return MessageBusCall<NMsgBusProxy::TBusSchemeOperation, NMsgBusProxy::TBusResponse>(config, request, + [](const NMsgBusProxy::TBusResponse& response) -> int { + if (response.Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { + Cerr << ToCString(static_cast<NMsgBusProxy::EResponseStatus>(response.Record.GetStatus())) << " " << response.Record.GetErrorReason() << Endl; + return 1; + } + return 0; + } + ); + } +}; + class TClientCommandSchemaUserAttributeDel: public TClientCommand { using TUserAttributesLimits = NSchemeShard::TUserAttributesLimits; @@ -1245,17 +1245,17 @@ public: } }; -class TClientCommandSchemaUserAttribute: public TClientCommandTree { -public: - TClientCommandSchemaUserAttribute() - : TClientCommandTree("user-attribute", { "ua" }, "User attribute operations") - { +class TClientCommandSchemaUserAttribute: public TClientCommandTree { +public: + TClientCommandSchemaUserAttribute() + : TClientCommandTree("user-attribute", { "ua" }, "User attribute operations") + { AddCommand(std::make_unique<TClientCommandSchemaUserAttributeGet>()); AddCommand(std::make_unique<TClientCommandSchemaUserAttributeSet>()); AddCommand(std::make_unique<TClientCommandSchemaUserAttributeDel>()); - } -}; - + } +}; + TClientCommandSchemaLite::TClientCommandSchemaLite() : TClientCommandTree("schema", {}, "Schema operations") { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp index 9932eaf36b9..8d8e54af65f 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp @@ -47,13 +47,13 @@ public: auto handler = [this](const NMsgBusProxy::TBusCmsResponse &response) -> int { auto &rec = response.Record; - switch (rec.GetStatus().GetCode()) { - case NKikimrCms::TStatus::OK: - case NKikimrCms::TStatus::ALLOW: - case NKikimrCms::TStatus::ALLOW_PARTIAL: - PrintOkResponse(rec); - return 0; - default: + switch (rec.GetStatus().GetCode()) { + case NKikimrCms::TStatus::OK: + case NKikimrCms::TStatus::ALLOW: + case NKikimrCms::TStatus::ALLOW_PARTIAL: + PrintOkResponse(rec); + return 0; + default: PrintErrorResponse(rec); return 1; } @@ -66,22 +66,22 @@ public: virtual void PrintErrorResponse(const NKikimrClient::TCmsResponse &response) { - PrintResponse(response); + PrintResponse(response); } virtual void PrintOkResponse(const NKikimrClient::TCmsResponse &response) { - PrintResponse(response); - } - - void PrintResponse(const NKikimrClient::TCmsResponse &response) - { - Cout << response.GetStatus().GetCode() << Endl - << response.DebugString(); - if (const auto& issues = response.GetStatus().GetReason()) { - Cerr << issues << Endl; - } - } + PrintResponse(response); + } + + void PrintResponse(const NKikimrClient::TCmsResponse &response) + { + Cout << response.GetStatus().GetCode() << Endl + << response.DebugString(); + if (const auto& issues = response.GetStatus().GetReason()) { + Cerr << issues << Endl; + } + } }; class TClientCommandState : public TCmsClientCommand { @@ -294,7 +294,7 @@ public: class TClientCommandWithAction : public TCmsClientCommand { public: enum EFreeField { - FF_TENANT, + FF_TENANT, FF_HOST, FF_SERVICE, FF_DEVICE @@ -311,7 +311,7 @@ public: NKikimrCms::TAction::EType type, EFreeField freeArgField, bool hasDuration) - : TCmsClientCommand(CommandName(freeArgField), {}, description) + : TCmsClientCommand(CommandName(freeArgField), {}, description) , Type(type) , FreeArgField(freeArgField) , HasDuration(hasDuration) @@ -325,14 +325,14 @@ public: Host = ""; Duration = 0; - if (FreeArgField == FF_SERVICE || FreeArgField == FF_DEVICE) + if (FreeArgField == FF_SERVICE || FreeArgField == FF_DEVICE) config.Opts->AddLongOption("host", "Host name").Required() .RequiredArgument("NAME").StoreResult(&Host); if (HasDuration) config.Opts->AddLongOption("duration", "Action duration in minutes") .Required().RequiredArgument("NUM").StoreResult(&Duration); config.SetFreeArgsMin(1); - config.Opts->SetFreeArgDefaultTitle("<NAME>", FreeArgDescr(FreeArgField)); + config.Opts->SetFreeArgDefaultTitle("<NAME>", FreeArgDescr(FreeArgField)); } void Parse(TConfig& config) override @@ -346,58 +346,58 @@ public: if (Host) action.SetHost(Host); if (param) { - switch (FreeArgField) { - case FF_TENANT: - action.SetTenant(param); - break; - case FF_HOST: + switch (FreeArgField) { + case FF_TENANT: + action.SetTenant(param); + break; + case FF_HOST: action.SetHost(param); - break; - case FF_SERVICE: - *action.AddServices() = param; - break; - case FF_DEVICE: + break; + case FF_SERVICE: + *action.AddServices() = param; + break; + case FF_DEVICE: *action.AddDevices() = param; - break; - default: - Y_FAIL("Unknown free arg field"); - } + break; + default: + Y_FAIL("Unknown free arg field"); + } } if (Duration) action.SetDuration(TDuration::Minutes(Duration).GetValue()); } } - -private: - static TString CommandName(EFreeField freeArgField) { - switch (freeArgField) { - case FF_TENANT: - return "tenant"; - case FF_HOST: - return "host"; - case FF_SERVICE: - return "service"; - case FF_DEVICE: - return "device"; - default: - Y_FAIL("Unknown free arg field"); - } - } - - static TString FreeArgDescr(EFreeField freeArgField) { - switch (freeArgField) { - case FF_TENANT: - return "Tenant name"; - case FF_HOST: - return "Host name"; - case FF_SERVICE: - return "Service name"; - case FF_DEVICE: - return "Device name"; - default: - Y_FAIL("Unknown free arg field"); - } - } + +private: + static TString CommandName(EFreeField freeArgField) { + switch (freeArgField) { + case FF_TENANT: + return "tenant"; + case FF_HOST: + return "host"; + case FF_SERVICE: + return "service"; + case FF_DEVICE: + return "device"; + default: + Y_FAIL("Unknown free arg field"); + } + } + + static TString FreeArgDescr(EFreeField freeArgField) { + switch (freeArgField) { + case FF_TENANT: + return "Tenant name"; + case FF_HOST: + return "Host name"; + case FF_SERVICE: + return "Service name"; + case FF_DEVICE: + return "Device name"; + default: + Y_FAIL("Unknown free arg field"); + } + } }; class TClientCommandMakeRequest : public TClientCommandWithAction { @@ -410,7 +410,7 @@ public: ui32 Hours; ui32 Minutes; TString TenantPolicy; - TString AvailabilityMode; + TString AvailabilityMode; TClientCommandMakeRequest(const TString &description, NKikimrCms::TAction::EType type, @@ -447,8 +447,8 @@ public: .RequiredArgument("none|default").StoreResult(&TenantPolicy); config.Opts->AddLongOption("allow-partial", "Allow partial permission") .NoArgument().SetFlag(&AllowPartial); - config.Opts->AddLongOption("availability-mode", "Availability mode") - .RequiredArgument("max|keep|force").DefaultValue("max").StoreResult(&AvailabilityMode); + config.Opts->AddLongOption("availability-mode", "Availability mode") + .RequiredArgument("max|keep|force").DefaultValue("max").StoreResult(&AvailabilityMode); } void Parse(TConfig& config) override @@ -475,16 +475,16 @@ public: else ythrow yexception() << "unknown tenant policy '" << TenantPolicy << "'"; } - if (AvailabilityMode) { - if (AvailabilityMode == "max") - rec.SetAvailabilityMode(NKikimrCms::MODE_MAX_AVAILABILITY); - else if (AvailabilityMode == "keep") - rec.SetAvailabilityMode(NKikimrCms::MODE_KEEP_AVAILABLE); - else if (AvailabilityMode == "force") - rec.SetAvailabilityMode(NKikimrCms::MODE_FORCE_RESTART); - else - ythrow yexception() << "unknown availability mode '" << AvailabilityMode << "'"; - } + if (AvailabilityMode) { + if (AvailabilityMode == "max") + rec.SetAvailabilityMode(NKikimrCms::MODE_MAX_AVAILABILITY); + else if (AvailabilityMode == "keep") + rec.SetAvailabilityMode(NKikimrCms::MODE_KEEP_AVAILABLE); + else if (AvailabilityMode == "force") + rec.SetAvailabilityMode(NKikimrCms::MODE_FORCE_RESTART); + else + ythrow yexception() << "unknown availability mode '" << AvailabilityMode << "'"; + } if (Hours || Minutes) { auto duration = TDuration::Minutes(Minutes) + TDuration::Hours(Hours); rec.SetDuration(duration.GetValue()); @@ -536,9 +536,9 @@ public: TClientCommandWithAction::FF_HOST, true)); AddCommand(std::make_unique<TClientCommandMakeRequest>("Ask for permission to restart tenant's hosts", - NKikimrCms::TAction::SHUTDOWN_HOST, - TClientCommandWithAction::FF_TENANT, - true)); + NKikimrCms::TAction::SHUTDOWN_HOST, + TClientCommandWithAction::FF_TENANT, + true)); } }; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index e167c0c9c5b..6766dd31719 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -424,7 +424,7 @@ protected: OPTION("pq-file", PQConfig); OPTION("pqcd-file", PQClusterDiscoveryConfig); OPTION("netclassifier-file", NetClassifierConfig); - OPTION("auth-file", AuthConfig); + OPTION("auth-file", AuthConfig); OPTION_MERGE("auth-token-file", AuthConfig); OPTION("key-file", KeyConfig); OPTION("pdisk-key-file", PDiskKeyConfig); @@ -943,7 +943,7 @@ protected: NodeResolveHost = NodeHost; THolder<NClient::TRegistrationResult> result; - while (!result || !result->IsSuccess()) { + while (!result || !result->IsSuccess()) { for (auto addr : addrs) { result = TryToRegisterDynamicNode(addr, domainName, NodeHost, NodeAddress, NodeResolveHost, GetSchemePath()); if (result->IsSuccess()) { @@ -1026,8 +1026,8 @@ protected: FQDNHostName(), TenantName, NodeType, - DeduceNodeDomain(), - AppConfig.GetAuthConfig().GetStaffApiUserToken()); + DeduceNodeDomain(), + AppConfig.GetAuthConfig().GetStaffApiUserToken()); if (!result.IsSuccess()) { error = result.GetErrorMessage(); diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index 610ed325370..faf1797413e 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -58,7 +58,7 @@ union TBasicKikimrServicesMask { bool EnablePersQueueClusterTracker:1; bool EnableSysViewService:1; bool EnableMeteringWriter:1; - bool EnableSchemeBoardMonitoring:1; + bool EnableSchemeBoardMonitoring:1; bool EnableConfigsCache:1; bool EnableLongTxService:1; bool EnableHealthCheckService:1; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 809fe55476a..819c1478d19 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -111,7 +111,7 @@ #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/tx/mediator/mediator.h> -#include <ydb/core/tx/replication/controller/controller.h> +#include <ydb/core/tx/replication/controller/controller.h> #include <ydb/core/tx/scheme_board/scheme_board.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/sequenceproxy/sequenceproxy.h> @@ -181,7 +181,7 @@ #include <library/cpp/lwtrace/mon/mon_lwtrace.h> #include <util/digest/city.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/generic/size_literals.h> #include <util/system/hostname.h> @@ -865,7 +865,7 @@ void TStateStorageServiceInitializer::InitializeServices(NActors::TActorSystemSe StartLocalStateStorageReplicas(CreateSchemeBoardReplica, sbrInfo.Get(), appData->SystemPoolId, *setup); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeStateStorageProxyID(ssid), - TActorSetupCmd(CreateStateStorageProxy(ssrInfo.Get(), ssbInfo.Get(), sbrInfo.Get()), + TActorSetupCmd(CreateStateStorageProxy(ssrInfo.Get(), ssbInfo.Get(), sbrInfo.Get()), TMailboxType::ReadAsFilled, appData->SystemPoolId))); } @@ -926,8 +926,8 @@ void TLocalServiceInitializer::InitializeServices( new TTabletSetupInfo(&CreateColumnShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.SequenceShard] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&NSequenceShard::CreateSequenceShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); - localConfig->TabletClassInfo[appData->DefaultTabletTypes.ReplicationController] = TLocalConfig::TTabletClassInfo( - new TTabletSetupInfo(&NReplication::CreateController, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); + localConfig->TabletClassInfo[appData->DefaultTabletTypes.ReplicationController] = TLocalConfig::TTabletClassInfo( + new TTabletSetupInfo(&NReplication::CreateController, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); TTenantPoolConfig::TPtr tenantPoolConfig = new TTenantPoolConfig(Config.GetTenantPoolConfig(), localConfig); if (!tenantPoolConfig->IsEnabled @@ -1878,10 +1878,10 @@ void TPersQueueLibSharedInstanceInitializer::InitializeServices(NActors::TActorS // TMemProfMonitorInitializer -TMemProfMonitorInitializer::TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - +TMemProfMonitorInitializer::TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + void TMemProfMonitorInitializer::InitializeServices( NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) @@ -1898,12 +1898,12 @@ void TMemProfMonitorInitializer::InitializeServices( appData->UserPoolId)); } -// TMemoryTrackerInitializer - -TMemoryTrackerInitializer::TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - +// TMemoryTrackerInitializer + +TMemoryTrackerInitializer::TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + void TMemoryTrackerInitializer::InitializeServices( NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) @@ -2216,17 +2216,17 @@ void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup *setup, co TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->IOPoolId))); } -TSchemeBoardMonitoringInitializer::TSchemeBoardMonitoringInitializer(const TKikimrRunConfig &runConfig) - : IKikimrServicesInitializer(runConfig) -{ -} - -void TSchemeBoardMonitoringInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { - setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( - MakeSchemeBoardMonitoringId(), - TActorSetupCmd(CreateSchemeBoardMonitoring(), TMailboxType::HTSwap, appData->UserPoolId))); -} - +TSchemeBoardMonitoringInitializer::TSchemeBoardMonitoringInitializer(const TKikimrRunConfig &runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TSchemeBoardMonitoringInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + MakeSchemeBoardMonitoringId(), + TActorSetupCmd(CreateSchemeBoardMonitoring(), TMailboxType::HTSwap, appData->UserPoolId))); +} + TYqlLogsInitializer::TYqlLogsInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index f0a6e5debd0..407ce1bb7ba 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -52,43 +52,43 @@ class TBasicServicesInitializer : public IKikimrServicesInitializer { public: TBasicServicesInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TImmediateControlBoardInitializer : public IKikimrServicesInitializer { public: TImmediateControlBoardInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TBSNodeWardenInitializer : public IKikimrServicesInitializer { public: TBSNodeWardenInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -// ss: proxies, replicas; board: replicas; scheme_board: replicas +// ss: proxies, replicas; board: replicas; scheme_board: replicas class TStateStorageServiceInitializer : public IKikimrServicesInitializer { public: TStateStorageServiceInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TLocalServiceInitializer : public IKikimrServicesInitializer { public: TLocalServiceInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TSharedCacheInitializer : public IKikimrServicesInitializer { public: - TSharedCacheInitializer(const TKikimrRunConfig& runConfig); + TSharedCacheInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TBlobCacheInitializer : public IKikimrServicesInitializer { @@ -108,35 +108,35 @@ public: TIntrusivePtr<NActors::NLog::TSettings> logSettings, std::shared_ptr<TLogBackend> logBackend); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TSchedulerActorInitializer : public IKikimrServicesInitializer { public: TSchedulerActorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TProfilerInitializer : public IKikimrServicesInitializer { public: TProfilerInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TResourceBrokerInitializer : public IKikimrServicesInitializer { public: TResourceBrokerInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TTabletResolverInitializer : public IKikimrServicesInitializer { +class TTabletResolverInitializer : public IKikimrServicesInitializer { public: TTabletResolverInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TTabletPipePeNodeCachesInitializer : public IKikimrServicesInitializer { @@ -146,25 +146,25 @@ public: void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TTabletMonitoringProxyInitializer : public IKikimrServicesInitializer { +class TTabletMonitoringProxyInitializer : public IKikimrServicesInitializer { public: TTabletMonitoringProxyInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TTabletCountersAggregatorInitializer : public IKikimrServicesInitializer { +class TTabletCountersAggregatorInitializer : public IKikimrServicesInitializer { public: TTabletCountersAggregatorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TGRpcProxyStatusInitializer : public IKikimrServicesInitializer { +class TGRpcProxyStatusInitializer : public IKikimrServicesInitializer { public: TGRpcProxyStatusInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TRestartsCountPublisher : public IKikimrServicesInitializer { @@ -174,14 +174,14 @@ class TRestartsCountPublisher : public IKikimrServicesInitializer { public: TRestartsCountPublisher(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TBootstrapperInitializer : public IKikimrServicesInitializer { +class TBootstrapperInitializer : public IKikimrServicesInitializer { public: TBootstrapperInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; // alternative (RTMR-style) bootstrapper @@ -192,23 +192,23 @@ public: TTabletsInitializer(const TKikimrRunConfig& runConfig, TIntrusivePtr<ITabletFactory> customTablets); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TMediatorTimeCastProxyInitializer : public IKikimrServicesInitializer { +class TMediatorTimeCastProxyInitializer : public IKikimrServicesInitializer { public: TMediatorTimeCastProxyInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TTxProxyInitializer : public IKikimrServicesInitializer { +class TTxProxyInitializer : public IKikimrServicesInitializer { TVector<ui64> CollectAllAllocatorsFromAllDomains(const NKikimr::TAppData* appData); public: TTxProxyInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TLongTxServiceInitializer : public IKikimrServicesInitializer { @@ -229,10 +229,10 @@ class TMiniKQLCompileServiceInitializer : public IKikimrServicesInitializer { public: TMiniKQLCompileServiceInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -// msgbus: proxy, trace +// msgbus: proxy, trace class TMessageBusServicesInitializer : public IKikimrServicesInitializer { NMsgBusProxy::IMessageBusServer& BusServer; @@ -240,7 +240,7 @@ public: TMessageBusServicesInitializer(const TKikimrRunConfig& runConfig, NMsgBusProxy::IMessageBusServer& busServer); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; // ticket_parser and so on... @@ -249,7 +249,7 @@ public: std::shared_ptr<TModuleFactories> Factories; TSecurityServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; // grpc_proxy @@ -260,7 +260,7 @@ private: public: TGRpcServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; #ifdef ACTORSLIB_COLLECT_EXEC_STATS @@ -269,7 +269,7 @@ class TStatsCollectorInitializer : public IKikimrServicesInitializer { public: TStatsCollectorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; #endif @@ -277,21 +277,21 @@ class TSelfPingInitializer : public IKikimrServicesInitializer { public: TSelfPingInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TWhiteBoardServiceInitializer : public IKikimrServicesInitializer { public: TWhiteBoardServiceInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TNodeIdentifierInitializer : public IKikimrServicesInitializer { public: TNodeIdentifierInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TTabletMonitorInitializer : public IKikimrServicesInitializer { @@ -303,7 +303,7 @@ public: const TIntrusivePtr<NNodeTabletMonitor::ITabletStateClassifier>& tabletStateClassifier, const TIntrusivePtr<NNodeTabletMonitor::ITabletListRenderer>& tabletListRenderer); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TViewerInitializer : public IKikimrServicesInitializer { @@ -312,69 +312,69 @@ class TViewerInitializer : public IKikimrServicesInitializer { public: TViewerInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TLoadInitializer : public IKikimrServicesInitializer { public: TLoadInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TFailureInjectionInitializer : public IKikimrServicesInitializer { public: TFailureInjectionInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; class TPersQueueL2CacheInitializer : public IKikimrServicesInitializer { public: TPersQueueL2CacheInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TNetClassifierInitializer : public IKikimrServicesInitializer { public: TNetClassifierInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TPersQueueClusterTrackerInitializer : public IKikimrServicesInitializer { public: TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer { public: TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TMemProfMonitorInitializer : public IKikimrServicesInitializer { public: - TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig); + TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TMemoryTrackerInitializer : public IKikimrServicesInitializer { public: - TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig); + TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TQuoterServiceInitializer : public IKikimrServicesInitializer { public: TQuoterServiceInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; @@ -382,33 +382,33 @@ class TKqpServiceInitializer : public IKikimrServicesInitializer { public: TKqpServiceInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; private: std::shared_ptr<TModuleFactories> Factories; }; -class TMemoryLogInitializer : public IKikimrServicesInitializer { - size_t LogBufferSize = 0; - size_t LogGrainSize = 0; - +class TMemoryLogInitializer : public IKikimrServicesInitializer { + size_t LogBufferSize = 0; + size_t LogGrainSize = 0; + public: TMemoryLogInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TCmsServiceInitializer : public IKikimrServicesInitializer { public: TCmsServiceInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TLeaseHolderInitializer : public IKikimrServicesInitializer { public: TLeaseHolderInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TSqsServiceInitializer : public IKikimrServicesInitializer { @@ -465,13 +465,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TSchemeBoardMonitoringInitializer : public IKikimrServicesInitializer { -public: - TSchemeBoardMonitoringInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +class TSchemeBoardMonitoringInitializer : public IKikimrServicesInitializer { +public: + TSchemeBoardMonitoringInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TYqlLogsInitializer : public IKikimrServicesInitializer { public: TYqlLogsInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index be8ee31f7f1..a4f74aa4e0f 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -503,8 +503,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["discovery"] = &hasDiscovery; bool hasTableService = services.empty(); names["table_service"] = &hasTableService; - bool hasSchemeService = false; - bool hasOperationService = false; + bool hasSchemeService = false; + bool hasOperationService = false; bool hasYql = false; names["yql"] = &hasYql; bool hasYqlInternal = services.empty(); @@ -525,9 +525,9 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["rate_limiter"] = &hasRateLimiter; bool hasLongTx = false; names["long_tx"] = &hasLongTx; - bool hasExport = services.empty(); + bool hasExport = services.empty(); names["export"] = &hasExport; - bool hasImport = services.empty(); + bool hasImport = services.empty(); names["import"] = &hasImport; bool hasAnalytics = false; names["analytics"] = &hasAnalytics; @@ -571,28 +571,28 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { } } - // dependencies - if (hasRateLimiter) { - hasKesus = true; - } - + // dependencies + if (hasRateLimiter) { + hasKesus = true; + } + if (hasYql) { hasTableService = true; hasYqlInternal = true; } - if (hasTableService || hasYqlInternal || hasPQ || hasKesus || hasPQv1 || hasExport || hasImport) { - hasSchemeService = true; - hasOperationService = true; - // backward compatability - hasExport = true; - } - - if (hasExport || hasImport) { - hasExport = true; - hasImport = true; - } - + if (hasTableService || hasYqlInternal || hasPQ || hasKesus || hasPQv1 || hasExport || hasImport) { + hasSchemeService = true; + hasOperationService = true; + // backward compatability + hasExport = true; + } + + if (hasExport || hasImport) { + hasExport = true; + hasImport = true; + } + for (const auto& [name, isEnabled] : names) { if (*isEnabled) { enabled.insert(name); @@ -653,20 +653,20 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { server.AddService(new NGRpcService::TGRpcYdbLongTxService(ActorSystem.Get(), Counters, grpcRequestProxyId)); } - if (hasSchemeService) { + if (hasSchemeService) { server.AddService(new NGRpcService::TGRpcYdbSchemeService(ActorSystem.Get(), Counters, grpcRequestProxyId)); - } - - if (hasOperationService) { - server.AddService(new NGRpcService::TGRpcOperationService(ActorSystem.Get(), Counters, grpcRequestProxyId)); - } - - if (hasExport) { - server.AddService(new NGRpcService::TGRpcYdbExportService(ActorSystem.Get(), Counters, grpcRequestProxyId)); - } - - if (hasImport) { - server.AddService(new NGRpcService::TGRpcYdbImportService(ActorSystem.Get(), Counters, grpcRequestProxyId)); + } + + if (hasOperationService) { + server.AddService(new NGRpcService::TGRpcOperationService(ActorSystem.Get(), Counters, grpcRequestProxyId)); + } + + if (hasExport) { + server.AddService(new NGRpcService::TGRpcYdbExportService(ActorSystem.Get(), Counters, grpcRequestProxyId)); + } + + if (hasImport) { + server.AddService(new NGRpcService::TGRpcYdbImportService(ActorSystem.Get(), Counters, grpcRequestProxyId)); } if (hasKesus) { @@ -910,10 +910,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->DataShardConfig = runConfig.AppConfig.GetDataShardConfig(); } - if (runConfig.AppConfig.HasMeteringConfig()) { - AppData->MeteringConfig = runConfig.AppConfig.GetMeteringConfig(); - } - + if (runConfig.AppConfig.HasMeteringConfig()) { + AppData->MeteringConfig = runConfig.AppConfig.GetMeteringConfig(); + } + // setup resource profiles AppData->ResourceProfiles = new TResourceProfiles; if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize()) @@ -1114,11 +1114,11 @@ void TKikimrRunner::InitializeActorSystem( } } - if (runConfig.AppConfig.HasGRpcConfig()) { - if (const ui32 grpcPort = runConfig.AppConfig.GetGRpcConfig().GetPort()) { + if (runConfig.AppConfig.HasGRpcConfig()) { + if (const ui32 grpcPort = runConfig.AppConfig.GetGRpcConfig().GetPort()) { auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort); - YdbDriver.Reset(new NYdb::TDriver(driverConfig)); - AppData->YdbDriver = YdbDriver.Get(); + YdbDriver.Reset(new NYdb::TDriver(driverConfig)); + AppData->YdbDriver = YdbDriver.Get(); } } @@ -1153,9 +1153,9 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers if (serviceMask.EnableBSNodeWarden) { sil->AddServiceInitializer(new TBSNodeWardenInitializer(runConfig)); } - if (serviceMask.EnableSchemeBoardMonitoring) { - sil->AddServiceInitializer(new TSchemeBoardMonitoringInitializer(runConfig)); - } + if (serviceMask.EnableSchemeBoardMonitoring) { + sil->AddServiceInitializer(new TSchemeBoardMonitoringInitializer(runConfig)); + } if (serviceMask.EnableStateStorageService) { sil->AddServiceInitializer(new TStateStorageServiceInitializer(runConfig)); } @@ -1437,8 +1437,8 @@ void TKikimrRunner::KikimrStop(bool graceful) { SqsHttp->Shutdown(); } - if (YdbDriver) { - YdbDriver->Stop(true); + if (YdbDriver) { + YdbDriver->Stop(true); } if (Monitoring) { diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index 0c9c8b92f98..dfd93a24e71 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -43,7 +43,7 @@ protected: bool GracefulShutdownSupported = false; THolder<NSQS::TAsyncHttpServer> SqsHttp; - THolder<NYdb::TDriver> YdbDriver; + THolder<NYdb::TDriver> YdbDriver; THolder<NKqp::TKqpShutdownController> KqpShutdownController; diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index ca55367731a..f410ef42f20 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -115,7 +115,7 @@ PEERDIR( ydb/core/tx/long_tx_service ydb/core/tx/long_tx_service/public ydb/core/tx/mediator - ydb/core/tx/replication/controller + ydb/core/tx/replication/controller ydb/core/tx/scheme_board ydb/core/tx/schemeshard ydb/core/tx/sequenceproxy diff --git a/ydb/core/engine/minikql/change_collector_iface.h b/ydb/core/engine/minikql/change_collector_iface.h index c71b3309533..fe6019b300e 100644 --- a/ydb/core/engine/minikql/change_collector_iface.h +++ b/ydb/core/engine/minikql/change_collector_iface.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/tablet_flat/flat_database.h> - -namespace NKikimr { -namespace NMiniKQL { - -class IChangeCollector { -public: - // basic change record's info - struct TChange: public std::tuple<ui64, TPathId, ui64> { - using std::tuple<ui64, TPathId, ui64>::tuple; - - ui64 Order() const { return std::get<0>(*this); } - const TPathId& PathId() const { return std::get<1>(*this); } - ui64 BodySize() const { return std::get<2>(*this); } - }; - -public: - virtual ~IChangeCollector() = default; - - virtual bool NeedToReadKeys() const = 0; - virtual void SetReadVersion(const TRowVersion& readVersion) = 0; - virtual void SetWriteVersion(const TRowVersion& writeVersion) = 0; - + +namespace NKikimr { +namespace NMiniKQL { + +class IChangeCollector { +public: + // basic change record's info + struct TChange: public std::tuple<ui64, TPathId, ui64> { + using std::tuple<ui64, TPathId, ui64>::tuple; + + ui64 Order() const { return std::get<0>(*this); } + const TPathId& PathId() const { return std::get<1>(*this); } + ui64 BodySize() const { return std::get<2>(*this); } + }; + +public: + virtual ~IChangeCollector() = default; + + virtual bool NeedToReadKeys() const = 0; + virtual void SetReadVersion(const TRowVersion& readVersion) = 0; + virtual void SetWriteVersion(const TRowVersion& writeVersion) = 0; + virtual bool Collect(const TTableId& tableId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) = 0; - - virtual const TVector<TChange>& GetCollected() const = 0; - virtual TVector<TChange>&& GetCollected() = 0; - virtual void Reset() = 0; -}; - -} // NMiniKQL -} // NKikimr + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) = 0; + + virtual const TVector<TChange>& GetCollected() const = 0; + virtual TVector<TChange>&& GetCollected() = 0; + virtual void Reset() = 0; +}; + +} // NMiniKQL +} // NKikimr diff --git a/ydb/core/engine/minikql/flat_local_minikql_host.h b/ydb/core/engine/minikql/flat_local_minikql_host.h index 88c31a0879b..edfd1822869 100644 --- a/ydb/core/engine/minikql/flat_local_minikql_host.h +++ b/ydb/core/engine/minikql/flat_local_minikql_host.h @@ -34,11 +34,11 @@ private: return Factory->GetReadVersion(tableId); } - IChangeCollector* GetChangeCollector(const TTableId& tableId) const override - { - return Factory->GetChangeCollector(tableId); - } - + IChangeCollector* GetChangeCollector(const TTableId& tableId) const override + { + return Factory->GetChangeCollector(tableId); + } + private: const TMiniKQLFactory* const Factory; }; diff --git a/ydb/core/engine/minikql/flat_local_tx_factory.cpp b/ydb/core/engine/minikql/flat_local_tx_factory.cpp index af5823b31df..d23f4d8bc73 100644 --- a/ydb/core/engine/minikql/flat_local_tx_factory.cpp +++ b/ydb/core/engine/minikql/flat_local_tx_factory.cpp @@ -39,11 +39,11 @@ TRowVersion TMiniKQLFactory::GetReadVersion(const TTableId& tableId) const return TRowVersion::Max(); } -IChangeCollector* TMiniKQLFactory::GetChangeCollector(const TTableId& tableId) const -{ - Y_UNUSED(tableId); - return nullptr; +IChangeCollector* TMiniKQLFactory::GetChangeCollector(const TTableId& tableId) const +{ + Y_UNUSED(tableId); + return nullptr; +} + } - } -} diff --git a/ydb/core/engine/minikql/flat_local_tx_factory.h b/ydb/core/engine/minikql/flat_local_tx_factory.h index 4a991ceb0cb..200d21b369a 100644 --- a/ydb/core/engine/minikql/flat_local_tx_factory.h +++ b/ydb/core/engine/minikql/flat_local_tx_factory.h @@ -1,7 +1,7 @@ #pragma once -#include "change_collector_iface.h" - +#include "change_collector_iface.h" + #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -17,7 +17,7 @@ namespace NMiniKQL { virtual TRowVersion GetWriteVersion(const TTableId& tableId) const; virtual TRowVersion GetReadVersion(const TTableId& tableId) const; - virtual IChangeCollector* GetChangeCollector(const TTableId& tableId) const; + virtual IChangeCollector* GetChangeCollector(const TTableId& tableId) const; }; } diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index 8d32f445459..36c35f32cc4 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -205,29 +205,29 @@ void TEngineHost::PinPages(const TVector<THolder<TKeyDesc>>& keys, ui64 pageFaul const TKeyDesc& key = *ki; if (TSysTables::IsSystemTable(key.TableId)) continue; - - TSet<TKeyDesc::EColumnOperation> columnOpFilter; - switch (key.RowOperation) { - case TKeyDesc::ERowOperation::Read: - columnOpFilter.insert(TKeyDesc::EColumnOperation::Read); - break; - case TKeyDesc::ERowOperation::Update: - case TKeyDesc::ERowOperation::Erase: { - const auto collector = GetChangeCollector(key.TableId); - if (collector && collector->NeedToReadKeys()) { - columnOpFilter.insert(TKeyDesc::EColumnOperation::Set); - columnOpFilter.insert(TKeyDesc::EColumnOperation::InplaceUpdate); - } - break; - } - default: - break; - } - - if (columnOpFilter.empty()) { + + TSet<TKeyDesc::EColumnOperation> columnOpFilter; + switch (key.RowOperation) { + case TKeyDesc::ERowOperation::Read: + columnOpFilter.insert(TKeyDesc::EColumnOperation::Read); + break; + case TKeyDesc::ERowOperation::Update: + case TKeyDesc::ERowOperation::Erase: { + const auto collector = GetChangeCollector(key.TableId); + if (collector && collector->NeedToReadKeys()) { + columnOpFilter.insert(TKeyDesc::EColumnOperation::Set); + columnOpFilter.insert(TKeyDesc::EColumnOperation::InplaceUpdate); + } + break; + } + default: + break; + } + + if (columnOpFilter.empty()) { continue; - } - + } + ui64 localTid = LocalTableId(key.TableId); Y_VERIFY(localTid, "table not exist"); const TScheme::TTableInfo* tableInfo = Scheme.GetTableInfo(localTid); @@ -238,7 +238,7 @@ void TEngineHost::PinPages(const TVector<THolder<TKeyDesc>>& keys, ui64 pageFaul TSmallVec<NTable::TTag> tags; for (const auto& column : key.Columns) { - if (columnOpFilter.contains(column.Operation)) { + if (columnOpFilter.contains(column.Operation)) { tags.push_back(column.Column); } } @@ -301,16 +301,16 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray Counters.InvisibleRowSkips += stats.Invisible; - if (NTable::EReady::Page == ready) { - if (auto collector = GetChangeCollector(tableId)) { - collector->Reset(); - } + if (NTable::EReady::Page == ready) { + if (auto collector = GetChangeCollector(tableId)) { + collector->Reset(); + } throw TNotReadyTabletException(); - } - - if (NTable::EReady::Gone == ready) { + } + + if (NTable::EReady::Gone == ready) { return NUdf::TUnboxedValue(); - } + } NUdf::TUnboxedValue* rowItems = nullptr; auto rowResult = holderFactory.CreateDirectArrayHolder(tags.size(), rowItems); @@ -571,12 +571,12 @@ public: List.EngineHost.GetCounters().SelectRangeDeletedRowSkips += std::exchange(Iter->Stats.DeletedRowSkips, 0); List.EngineHost.GetCounters().InvisibleRowSkips += std::exchange(Iter->Stats.InvisibleRowSkips, 0); - if (Iter->Last() == NTable::EReady::Page) { - if (auto collector = List.EngineHost.GetChangeCollector(List.TableId)) { - collector->Reset(); - } + if (Iter->Last() == NTable::EReady::Page) { + if (auto collector = List.EngineHost.GetChangeCollector(List.TableId)) { + collector->Reset(); + } throw TNotReadyTabletException(); - } + } if (List.Truncated || List.SizeBytes) { Y_VERIFY_DEBUG(List.Truncated && *List.Truncated == truncated); @@ -869,18 +869,18 @@ void TEngineHost::UpdateRow(const TTableId& tableId, const TArrayRef<const TCell valueBytes += upd.Value.IsNull() ? 1 : upd.Value.Size(); } - if (auto collector = GetChangeCollector(tableId)) { - collector->SetWriteVersion(GetWriteVersion(tableId)); - if (collector->NeedToReadKeys()) { - collector->SetReadVersion(GetReadVersion(tableId)); - } - + if (auto collector = GetChangeCollector(tableId)) { + collector->SetWriteVersion(GetWriteVersion(tableId)); + if (collector->NeedToReadKeys()) { + collector->SetReadVersion(GetReadVersion(tableId)); + } + if (!collector->Collect(tableId, NTable::ERowOp::Upsert, key, ops)) { - collector->Reset(); - throw TNotReadyTabletException(); - } - } - + collector->Reset(); + throw TNotReadyTabletException(); + } + } + Db.Update(localTid, NTable::ERowOp::Upsert, key, ops, GetWriteVersion(tableId)); Settings.KeyAccessSampler->AddSample(tableId, row); @@ -897,18 +897,18 @@ void TEngineHost::EraseRow(const TTableId& tableId, const TArrayRef<const TCell> ui64 keyBytes = 0; ConvertTableKeys(Scheme, tableInfo, row, key, &keyBytes); - if (auto collector = GetChangeCollector(tableId)) { - collector->SetWriteVersion(GetWriteVersion(tableId)); - if (collector->NeedToReadKeys()) { - collector->SetReadVersion(GetReadVersion(tableId)); - } - + if (auto collector = GetChangeCollector(tableId)) { + collector->SetWriteVersion(GetWriteVersion(tableId)); + if (collector->NeedToReadKeys()) { + collector->SetReadVersion(GetReadVersion(tableId)); + } + if (!collector->Collect(tableId, NTable::ERowOp::Erase, key, {})) { - collector->Reset(); - throw TNotReadyTabletException(); - } - } - + collector->Reset(); + throw TNotReadyTabletException(); + } + } + Db.Update(localTid, NTable::ERowOp::Erase, key, { }, GetWriteVersion(tableId)); Settings.KeyAccessSampler->AddSample(tableId, row); diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h index 5748e7f1f99..012ee6891bc 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.h +++ b/ydb/core/engine/minikql/minikql_engine_host.h @@ -1,7 +1,7 @@ #pragma once -#include "change_collector_iface.h" - +#include "change_collector_iface.h" + #include <util/generic/cast.h> #include <ydb/core/tablet/tablet_exception.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> @@ -127,8 +127,8 @@ public: virtual TRowVersion GetWriteVersion(const TTableId& tableId) const = 0; virtual TRowVersion GetReadVersion(const TTableId& tableId) const = 0; - virtual IChangeCollector* GetChangeCollector(const TTableId& tableId) const = 0; - + virtual IChangeCollector* GetChangeCollector(const TTableId& tableId) const = 0; + protected: virtual ui64 LocalTableId(const TTableId& tableId) const; void ConvertKeys(const TScheme::TTableInfo* tableInfo, const TArrayRef<const TCell>& row, @@ -156,11 +156,11 @@ public: Y_UNUSED(tableId); return TRowVersion::Max(); } - - IChangeCollector* GetChangeCollector(const TTableId& tableId) const override { - Y_UNUSED(tableId); - return nullptr; - } + + IChangeCollector* GetChangeCollector(const TTableId& tableId) const override { + Y_UNUSED(tableId); + return nullptr; + } }; void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags); diff --git a/ydb/core/engine/mkql_engine_flat.cpp b/ydb/core/engine/mkql_engine_flat.cpp index 6e13b831e96..c7fe388eef8 100644 --- a/ydb/core/engine/mkql_engine_flat.cpp +++ b/ydb/core/engine/mkql_engine_flat.cpp @@ -1069,14 +1069,14 @@ public: const auto& myKeysStruct = static_cast<const TStructLiteral&>(*myKeys.GetNode()); MKQL_ENSURE(myKeysStruct.GetValuesCount() == 2, "Expected 2 members"); - // 0 - reads, 1 - writes - for (ui32 opId = 0; opId <= 1; ++opId) { - auto myOps = myKeysStruct.GetValue(opId); - MKQL_ENSURE(myOps.IsImmediate() && myOps.GetNode()->GetType()->IsStruct(), "Expected immediate struct"); - - auto myOpsStruct = static_cast<TStructLiteral*>(myOps.GetNode()); - for (ui32 i = 0, e = myOpsStruct->GetValuesCount(); i < e; ++i) { - TRuntimeNode item = myOpsStruct->GetValue(i); + // 0 - reads, 1 - writes + for (ui32 opId = 0; opId <= 1; ++opId) { + auto myOps = myKeysStruct.GetValue(opId); + MKQL_ENSURE(myOps.IsImmediate() && myOps.GetNode()->GetType()->IsStruct(), "Expected immediate struct"); + + auto myOpsStruct = static_cast<TStructLiteral*>(myOps.GetNode()); + for (ui32 i = 0, e = myOpsStruct->GetValuesCount(); i < e; ++i) { + TRuntimeNode item = myOpsStruct->GetValue(i); Y_VERIFY(item.GetNode()->GetType()->IsCallable(), "Bad shard program"); THolder<TKeyDesc> desc = ExtractTableKey(*static_cast<TCallable*>(item.GetNode()), Strings, Env); Y_VERIFY(desc); diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index f3e2622bb53..44b25c4a5f9 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -2,7 +2,7 @@ #include <grpc++/support/byte_buffer.h> #include <grpc++/support/slice.h> - + #include <library/cpp/grpc/server/grpc_request_base.h> #include <library/cpp/string_utils/quote/quote.h> @@ -44,7 +44,7 @@ namespace NGRpcService { using TYdbIssueMessageType = Ydb::Issue::IssueMessage; -std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TString& path); +std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TString& path); std::pair<TString, TString> SplitPath(const TString& path); void RefreshToken(const TString& token, const TString& database, const TActorContext& ctx, TActorId id); @@ -57,7 +57,7 @@ struct TRpcServices { EvDropTable, EvCopyTable, EvCopyTables, - EvListDirectory, + EvListDirectory, EvRenameTables, EvDescribeTable, EvDescribePath, @@ -101,9 +101,9 @@ struct TRpcServices { EvExperimentalStreamQuery, EvStreamPQWrite, EvStreamPQRead, - EvPQReadInfo, - EvListOperations, - EvExportToYt, + EvPQReadInfo, + EvListOperations, + EvExportToYt, EvDiscoverPQClusters, EvBulkUpsert, EvWhoAmI, @@ -117,7 +117,7 @@ struct TRpcServices { EvKikhouseCreateSnapshot, EvKikhouseRefreshSnapshot, EvKikhouseDiscardSnapshot, - EvExportToS3, + EvExportToS3, EvSelfCheck, EvStreamExecuteScanQuery, EvPQDropTopic, @@ -126,17 +126,17 @@ struct TRpcServices { EvPQDescribeTopic, EvPQAddReadRule, EvPQRemoveReadRule, - EvGetDiskSpaceUsage, + EvGetDiskSpaceUsage, EvStopServingDatabase, EvCoordinationSession, - EvImportFromS3, + EvImportFromS3, EvLongTxBegin, EvLongTxCommit, EvLongTxRollback, EvLongTxWrite, EvLongTxRead, EvExplainYqlScript, - EvImportData, + EvImportData, EvAnalyticsReserved, EvDataStreamsCreateStream, EvDataStreamsDeleteStream, @@ -328,8 +328,8 @@ public: virtual const NGrpc::TAuthState& GetAuthState() const = 0; virtual void ReplyUnauthenticated(const TString& msg = "") = 0; virtual void ReplyUnavaliable() = 0; - - virtual bool Validate(TString& error) = 0; + + virtual bool Validate(TString& error) = 0; virtual void UseDatabase(const TString& database) = 0; // This method allows to set hook for unary call. @@ -475,10 +475,10 @@ public: IssueManager_.RaiseIssues(issues); } - bool Validate(TString&) override { - return true; - } - + bool Validate(TString&) override { + return true; + } + void UseDatabase(const TString& database) override { Y_UNUSED(database); } @@ -634,10 +634,10 @@ public: return Ctx_->GetPeerName(); } - bool Validate(TString&) override { - return true; - } - + bool Validate(TString&) override { + return true; + } + void UseDatabase(const TString& database) override { Ctx_->UseDatabase(database); } @@ -786,7 +786,7 @@ public: }; template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, typename TDerived> -class TGRpcRequestWrapperImpl : +class TGRpcRequestWrapperImpl : public std::conditional_t<IsOperation, TGrpcResponseSenderImpl<TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TDerived>>, IRequestNoOpCtx>, @@ -859,10 +859,10 @@ public: return ToMaybe(Ctx_->GetPeerMetaValues(key)); } - bool Validate(TString&) override { - return true; - } - + bool Validate(TString&) override { + return true; + } + void UseDatabase(const TString& database) override { Ctx_->UseDatabase(database); } @@ -871,8 +871,8 @@ public: TResponse* resp = CreateResponseMessage(); TCommonResponseFiller<TResp, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, Ydb::StatusIds::UNAVAILABLE); Reply(resp, Ydb::StatusIds::UNAVAILABLE); - } - + } + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { TResponse* resp = CreateResponseMessage(); TCommonResponseFiller<TResponse, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, status); @@ -1072,10 +1072,10 @@ template <typename TReq, typename TResp> using TGrpcRequestNoOperationCall = TGrpcRequestCall<TReq, TResp, false>; template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> -class TGRpcRequestWrapper : +class TGRpcRequestWrapper : public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { -public: +public: static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); static constexpr bool IsOp = IsOperation; static constexpr TRateLimiterMode RateLimitMode = RlMode; @@ -1083,7 +1083,7 @@ public: TGRpcRequestWrapper(NGrpc::IRequestContextBase* ctx) : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) - { } + { } TRateLimiterMode GetRlMode() const override { return RateLimitMode; @@ -1092,8 +1092,8 @@ public: bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult&, ICheckerIface*) override { return false; } -}; - +}; + template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation = true, TRateLimiterMode RlMode = TRateLimiterMode::Off> class TGRpcRequestWrapperNoAuth : public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestWrapperNoAuth<TRpcId, TReq, TResp, IsOperation, RlMode>> { @@ -1121,17 +1121,17 @@ public: }; template<ui32 TRpcId, typename TReq, typename TResp, bool IsOperation, TRateLimiterMode RlMode = TRateLimiterMode::Off> -class TGRpcRequestValidationWrapper : +class TGRpcRequestValidationWrapper : public TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>> { - -public: + +public: static IActor* CreateRpcActor(typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type* msg); static constexpr bool IsOp = IsOperation; static constexpr TRateLimiterMode RateLimitMode = RlMode; TGRpcRequestValidationWrapper(NGrpc::IRequestContextBase* ctx) : TGRpcRequestWrapperImpl<TRpcId, TReq, TResp, IsOperation, TGRpcRequestValidationWrapper<TRpcId, TReq, TResp, IsOperation, RlMode>>(ctx) - { } - + { } + TRateLimiterMode GetRlMode() const override { return RateLimitMode; } @@ -1140,10 +1140,10 @@ public: return false; } - bool Validate(TString& error) override { - return this->GetProtoRequest()->validate(error); - } -}; - + bool Validate(TString& error) override { + return this->GetProtoRequest()->validate(error); + } +}; + } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp index c8dcb5adbd2..a1e7accb877 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.cpp +++ b/ydb/core/grpc_services/grpc_request_proxy.cpp @@ -298,7 +298,7 @@ private: void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe()); - AllowYdbRequestsWithoutDatabase = AppData(ctx)->FeatureFlags.GetAllowYdbRequestsWithoutDatabase(); + AllowYdbRequestsWithoutDatabase = AppData(ctx)->FeatureFlags.GetAllowYdbRequestsWithoutDatabase(); // Subscribe for TableService config changes ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem; diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index 12224cc8a3b..97315f6e9fb 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -55,9 +55,9 @@ protected: void Handle(TEvRenameTablesRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDescribeTableRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvGetOperationRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvCancelOperationRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvForgetOperationRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvListOperationsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvCancelOperationRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvForgetOperationRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvListOperationsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCreateSessionRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvKeepAliveRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDeleteSessionRequest::TPtr& ev, const TActorContext& ctx); @@ -96,10 +96,10 @@ protected: void Handle(TEvPQAddReadRuleRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQRemoveReadRuleRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQDescribeTopicRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExportToYtRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExportToYtRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvBulkUpsertRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/grpc_services/rpc_alter_coordination_node.cpp b/ydb/core/grpc_services/rpc_alter_coordination_node.cpp index b5e1babcd8f..52cfe8c9403 100644 --- a/ydb/core/grpc_services/rpc_alter_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_alter_coordination_node.cpp @@ -38,7 +38,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_alter_table.cpp b/ydb/core/grpc_services/rpc_alter_table.cpp index 32f513b9919..d1f2755b66d 100644 --- a/ydb/core/grpc_services/rpc_alter_table.cpp +++ b/ydb/core/grpc_services/rpc_alter_table.cpp @@ -12,8 +12,8 @@ #include <ydb/core/ydb_convert/column_families.h> #include <ydb/core/ydb_convert/table_description.h> -#include <util/generic/hash_set.h> - +#include <util/generic/hash_set.h> + #define TXLOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) #define TXLOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) #define TXLOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, LogPrefix << stream) @@ -49,32 +49,32 @@ static bool CheckAccess(const NACLib::TUserToken& userToken, const NSchemeCache: return true; } -static std::pair<StatusIds::StatusCode, TString> CheckAddIndexDesc(const Ydb::Table::TableIndex& desc) { - if (!desc.name()) { - return {StatusIds::BAD_REQUEST, "Index must have a name"}; - } - +static std::pair<StatusIds::StatusCode, TString> CheckAddIndexDesc(const Ydb::Table::TableIndex& desc) { + if (!desc.name()) { + return {StatusIds::BAD_REQUEST, "Index must have a name"}; + } + if (!desc.index_columns_size()) { - return {StatusIds::BAD_REQUEST, "At least one column must be specified"}; + return {StatusIds::BAD_REQUEST, "At least one column must be specified"}; } - - if (!desc.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { - return {StatusIds::UNSUPPORTED, "Data column feature is not supported yet"}; + + if (!desc.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { + return {StatusIds::UNSUPPORTED, "Data column feature is not supported yet"}; + } + + switch (desc.type_case()) { + case Table::TableIndex::kGlobalIndex: + break; + case Table::TableIndex::kGlobalAsyncIndex: + if (!AppData()->FeatureFlags.GetEnableAsyncIndexes()) { + return {StatusIds::UNSUPPORTED, "Async indexes are not supported yet"}; + } + break; + default: + break; } - - switch (desc.type_case()) { - case Table::TableIndex::kGlobalIndex: - break; - case Table::TableIndex::kGlobalAsyncIndex: - if (!AppData()->FeatureFlags.GetEnableAsyncIndexes()) { - return {StatusIds::UNSUPPORTED, "Async indexes are not supported yet"}; - } - break; - default: - break; - } - - return {StatusIds::SUCCESS, ""}; + + return {StatusIds::SUCCESS, ""}; } class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTableRequest> { @@ -87,47 +87,47 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab } IActor::PassAway(); } - - enum class EOp { - // columns, column families, storage, ttl - Common, + + enum class EOp { + // columns, column families, storage, ttl + Common, // add indices AddIndex, // drop indices DropIndex, - // add/alter/drop attributes - Attribute, - }; - - THashSet<EOp> GetOps() const { + // add/alter/drop attributes + Attribute, + }; + + THashSet<EOp> GetOps() const { const auto& req = GetProtoRequest(); - THashSet<EOp> ops; - - if (req->add_columns_size() || req->drop_columns_size() || req->alter_columns_size() - || req->ttl_action_case() != Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET - || req->has_alter_storage_settings() + THashSet<EOp> ops; + + if (req->add_columns_size() || req->drop_columns_size() || req->alter_columns_size() + || req->ttl_action_case() != Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET + || req->has_alter_storage_settings() || req->add_column_families_size() || req->alter_column_families_size() || req->set_compaction_policy() || req->has_alter_partitioning_settings() || req->set_key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED || req->has_set_read_replicas_settings()) { - ops.emplace(EOp::Common); - } - + ops.emplace(EOp::Common); + } + if (req->add_indexes_size()) { ops.emplace(EOp::AddIndex); - } - + } + if (req->drop_indexes_size()) { ops.emplace(EOp::DropIndex); } - if (req->alter_attributes_size()) { - ops.emplace(EOp::Attribute); - } - - return ops; - } - + if (req->alter_attributes_size()) { + ops.emplace(EOp::Attribute); + } + + return ops; + } + public: TAlterTableRPC(IRequestOpCtx* msg) : TBase(msg) {} @@ -139,40 +139,40 @@ public: if (!Request_->GetInternalToken().empty()) { UserToken = MakeHolder<NACLib::TUserToken>(Request_->GetInternalToken()); } - - auto ops = GetOps(); - if (!ops) { - return Reply(StatusIds::BAD_REQUEST, "Empty alter", - NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } - if (ops.size() != 1) { - return Reply(StatusIds::UNSUPPORTED, "Mixed alter is unsupported", - NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } - - switch (*ops.begin()) { - case EOp::Common: + + auto ops = GetOps(); + if (!ops) { + return Reply(StatusIds::BAD_REQUEST, "Empty alter", + NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); + } + if (ops.size() != 1) { + return Reply(StatusIds::UNSUPPORTED, "Mixed alter is unsupported", + NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); + } + + switch (*ops.begin()) { + case EOp::Common: // Altering table settings will need table profiles SendConfigRequest(ctx); ctx.Schedule(TDuration::Seconds(15), new TEvents::TEvWakeup(WakeupTagGetConfig)); Become(&TAlterTableRPC::AlterStateGetConfig); return; - + case EOp::AddIndex: if (req->add_indexes_size() == 1) { const auto& index = req->add_indexes(0); - auto [status, issues] = CheckAddIndexDesc(index); - if (status == StatusIds::SUCCESS) { + auto [status, issues] = CheckAddIndexDesc(index); + if (status == StatusIds::SUCCESS) { PrepareAlterTableAddIndex(); } else { - return Reply(status, issues, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); + return Reply(status, issues, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); } - } else { + } else { return Reply(StatusIds::UNSUPPORTED, "Only one index can be added by one operation", - NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); - } - break; - + NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); + } + break; + case EOp::DropIndex: if (req->drop_indexes_size() == 1) { DropIndex(ctx); @@ -182,11 +182,11 @@ public: } break; - case EOp::Attribute: - AlterUserAttributes(ctx); - break; - } - + case EOp::Attribute: + AlterUserAttributes(ctx); + break; + } + Become(&TAlterTableRPC::AlterStateWork); } @@ -397,7 +397,7 @@ private: ctx.Send(MakeTxProxyID(), proposeRequest.release()); } - void AlterTable(const TActorContext &ctx) { + void AlterTable(const TActorContext &ctx) { const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { @@ -478,7 +478,7 @@ private: desc->ClearPartitionConfig(); } - if (!FillAlterTableSettingsDesc(*desc, *req, Profiles, code, error, AppData())) { + if (!FillAlterTableSettingsDesc(*desc, *req, Profiles, code, error, AppData())) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); return Reply(code, issues, ctx); @@ -487,40 +487,40 @@ private: ctx.Send(MakeTxProxyID(), proposeRequest.release()); } - void AlterUserAttributes(const TActorContext &ctx) { + void AlterUserAttributes(const TActorContext &ctx) { const auto req = GetProtoRequest(); - - std::pair<TString, TString> pathPair; - try { - pathPair = SplitPath(req->path()); - } catch (const std::exception&) { - return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); - } - - const auto& workingDir = pathPair.first; - const auto& name = pathPair.second; - - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); - auto& record = proposeRequest->Record; - auto& modifyScheme = *record.MutableTransaction()->MutableModifyScheme(); - - modifyScheme.SetWorkingDir(workingDir); + + std::pair<TString, TString> pathPair; + try { + pathPair = SplitPath(req->path()); + } catch (const std::exception&) { + return ReplyWithStatus(StatusIds::BAD_REQUEST, ctx); + } + + const auto& workingDir = pathPair.first; + const auto& name = pathPair.second; + + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + auto& record = proposeRequest->Record; + auto& modifyScheme = *record.MutableTransaction()->MutableModifyScheme(); + + modifyScheme.SetWorkingDir(workingDir); modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes); - - auto& alter = *modifyScheme.MutableAlterUserAttributes(); - alter.SetPathName(name); - - for (auto [key, value] : req->alter_attributes()) { - auto& attr = *alter.AddUserAttributes(); - attr.SetKey(key); - if (value) { - attr.SetValue(value); - } - } - - ctx.Send(MakeTxProxyID(), proposeRequest.release()); - } - + + auto& alter = *modifyScheme.MutableAlterUserAttributes(); + alter.SetPathName(name); + + for (auto [key, value] : req->alter_attributes()) { + auto& attr = *alter.AddUserAttributes(); + attr.SetKey(key); + if (value) { + attr.SetValue(value); + } + } + + ctx.Send(MakeTxProxyID(), proposeRequest.release()); + } + void ReplyWithStatus(StatusIds::StatusCode status, const TActorContext &ctx) { Request_->ReplyWithYdbStatus(status); diff --git a/ydb/core/grpc_services/rpc_calls.cpp b/ydb/core/grpc_services/rpc_calls.cpp index 6c6795e21bf..b94d168147c 100644 --- a/ydb/core/grpc_services/rpc_calls.cpp +++ b/ydb/core/grpc_services/rpc_calls.cpp @@ -2,7 +2,7 @@ #include "grpc_request_proxy.h" #include <ydb/core/base/path.h> - + namespace NKikimr { namespace NGRpcService { @@ -32,17 +32,17 @@ void FillYdbStatus(Ydb::Coordination::SessionResponse& resp, const NYql::TIssues NYql::IssuesToMessage(issues, failure->mutable_issues()); } -std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TString& path) { - std::pair<TString, TString> pathPair; - TString error; - - if (!TrySplitPathByDb(path, database.GetOrElse(TString()), pathPair, error)) { - ythrow yexception() << error; - } - - return pathPair; -} - +std::pair<TString, TString> SplitPath(const TMaybe<TString>& database, const TString& path) { + std::pair<TString, TString> pathPair; + TString error; + + if (!TrySplitPathByDb(path, database.GetOrElse(TString()), pathPair, error)) { + ythrow yexception() << error; + } + + return pathPair; +} + std::pair<TString, TString> SplitPath(const TString& path) { auto splitPos = path.find_last_of('/'); if (splitPos == path.npos || splitPos + 1 == path.size()) { diff --git a/ydb/core/grpc_services/rpc_calls.h b/ydb/core/grpc_services/rpc_calls.h index 68570ec911a..43136c2f4ad 100644 --- a/ydb/core/grpc_services/rpc_calls.h +++ b/ydb/core/grpc_services/rpc_calls.h @@ -4,7 +4,7 @@ #include "local_rate_limiter.h" #include <ydb/core/grpc_services/base/base.h> - + #include <ydb/public/api/protos/ydb_auth.pb.h> #include <ydb/public/api/protos/ydb_clickhouse_internal.pb.h> #include <ydb/public/api/protos/ydb_cms.pb.h> @@ -27,7 +27,7 @@ #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.pb.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <util/generic/maybe.h> namespace NKikimr { diff --git a/ydb/core/grpc_services/rpc_calls_ut.cpp b/ydb/core/grpc_services/rpc_calls_ut.cpp index d50560bb552..514be6ffee5 100644 --- a/ydb/core/grpc_services/rpc_calls_ut.cpp +++ b/ydb/core/grpc_services/rpc_calls_ut.cpp @@ -4,35 +4,35 @@ namespace NKikimr { -Y_UNIT_TEST_SUITE(SplitPathTests) { - Y_UNIT_TEST(WithoutDatabaseShouldSuccess) { +Y_UNIT_TEST_SUITE(SplitPathTests) { + Y_UNIT_TEST(WithoutDatabaseShouldSuccess) { { - auto pair = NGRpcService::SplitPath("/root/table"); - UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root"); + auto pair = NGRpcService::SplitPath("/root/table"); + UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root"); UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); } - { - auto pair = NGRpcService::SplitPath("/root/dir/table"); - UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root/dir"); - UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); - } + { + auto pair = NGRpcService::SplitPath("/root/dir/table"); + UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root/dir"); + UNIT_ASSERT_VALUES_EQUAL(pair.second, "table"); + } + } + + Y_UNIT_TEST(WithDatabaseShouldSuccess) { + for (auto db : TVector<TString>{"/root/db", "/root/db/"}) { + for (auto subpath : TVector<TString>{"table", "dir/table"}) { + auto pair = NGRpcService::SplitPath(db, "/root/db/" + subpath); + UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root/db"); + UNIT_ASSERT_VALUES_EQUAL(pair.second, subpath); + } + } + } + + Y_UNIT_TEST(WithDatabaseShouldFail) { + for (auto path : TVector<TString>{"/root/db1/table", "/root/db1/dir/table"}) { + UNIT_ASSERT_EXCEPTION(NGRpcService::SplitPath("/root/db2", path), yexception); + } } - - Y_UNIT_TEST(WithDatabaseShouldSuccess) { - for (auto db : TVector<TString>{"/root/db", "/root/db/"}) { - for (auto subpath : TVector<TString>{"table", "dir/table"}) { - auto pair = NGRpcService::SplitPath(db, "/root/db/" + subpath); - UNIT_ASSERT_VALUES_EQUAL(pair.first, "/root/db"); - UNIT_ASSERT_VALUES_EQUAL(pair.second, subpath); - } - } - } - - Y_UNIT_TEST(WithDatabaseShouldFail) { - for (auto path : TVector<TString>{"/root/db1/table", "/root/db1/dir/table"}) { - UNIT_ASSERT_EXCEPTION(NGRpcService::SplitPath("/root/db2", path), yexception); - } - } } } // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_cancel_operation.cpp b/ydb/core/grpc_services/rpc_cancel_operation.cpp index 26431fa28c1..579f927c80f 100644 --- a/ydb/core/grpc_services/rpc_cancel_operation.cpp +++ b/ydb/core/grpc_services/rpc_cancel_operation.cpp @@ -1,127 +1,127 @@ -#include "grpc_request_proxy.h" -#include "operation_helpers.h" -#include "rpc_calls.h" -#include "rpc_operation_request_base.h" - +#include "grpc_request_proxy.h" +#include "operation_helpers.h" +#include "rpc_calls.h" +#include "rpc_operation_request_base.h" + #include <ydb/core/tx/schemeshard/schemeshard_build_index.h> #include <ydb/core/tx/schemeshard/schemeshard_export.h> #include <ydb/core/tx/schemeshard/schemeshard_import.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NSchemeShard; -using namespace NKikimrIssues; -using namespace NOperationId; -using namespace Ydb; - -class TCancelOperationRPC: public TRpcOperationRequestActor<TCancelOperationRPC, TEvCancelOperationRequest> { - TStringBuf GetLogPrefix() const override { - switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - return "[CancelExport]"; - case TOperationId::IMPORT: - return "[CancelImport]"; - case TOperationId::BUILD_INDEX: - return "[CancelIndexBuild]"; - default: - return "[Untagged]"; - } - } - - IEventBase* MakeRequest() override { - switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - return new TEvExport::TEvCancelExportRequest(TxId, DatabaseName, RawOperationId); - case TOperationId::IMPORT: - return new TEvImport::TEvCancelImportRequest(TxId, DatabaseName, RawOperationId); - case TOperationId::BUILD_INDEX: - return new TEvIndexBuilder::TEvCancelRequest(TxId, DatabaseName, RawOperationId); - default: - Y_FAIL("unreachable"); - } - } - - void Handle(TEvExport::TEvCancelExportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvCancelExportResponse" - << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); - } - - void Handle(TEvImport::TEvCancelImportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvImport::TEvCancelImportResponse" - << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); +using namespace NKikimrIssues; +using namespace NOperationId; +using namespace Ydb; + +class TCancelOperationRPC: public TRpcOperationRequestActor<TCancelOperationRPC, TEvCancelOperationRequest> { + TStringBuf GetLogPrefix() const override { + switch (OperationId.GetKind()) { + case TOperationId::EXPORT: + return "[CancelExport]"; + case TOperationId::IMPORT: + return "[CancelImport]"; + case TOperationId::BUILD_INDEX: + return "[CancelIndexBuild]"; + default: + return "[Untagged]"; + } + } + + IEventBase* MakeRequest() override { + switch (OperationId.GetKind()) { + case TOperationId::EXPORT: + return new TEvExport::TEvCancelExportRequest(TxId, DatabaseName, RawOperationId); + case TOperationId::IMPORT: + return new TEvImport::TEvCancelImportRequest(TxId, DatabaseName, RawOperationId); + case TOperationId::BUILD_INDEX: + return new TEvIndexBuilder::TEvCancelRequest(TxId, DatabaseName, RawOperationId); + default: + Y_FAIL("unreachable"); + } + } + + void Handle(TEvExport::TEvCancelExportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvCancelExportResponse" + << ": record# " << record.ShortDebugString()); + + Reply(record.GetStatus(), record.GetIssues()); + } + + void Handle(TEvImport::TEvCancelImportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvImport::TEvCancelImportResponse" + << ": record# " << record.ShortDebugString()); + + Reply(record.GetStatus(), record.GetIssues()); } - void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; + void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; - LOG_D("Handle TEvIndexBuilder::TEvCancelResponse" - << ": record# " << record.ShortDebugString()); + LOG_D("Handle TEvIndexBuilder::TEvCancelResponse" + << ": record# " << record.ShortDebugString()); Reply(record.GetStatus(), record.GetIssues()); } -public: - using TRpcOperationRequestActor::TRpcOperationRequestActor; - - void Bootstrap() { - const TString& id = Request->GetProtoRequest()->id(); - - try { +public: + using TRpcOperationRequestActor::TRpcOperationRequestActor; + + void Bootstrap() { + const TString& id = Request->GetProtoRequest()->id(); + + try { OperationId = TOperationId(id); - + switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - case TOperationId::IMPORT: - case TOperationId::BUILD_INDEX: - if (!TryGetId(OperationId, RawOperationId)) { - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unable to extract operation id"); - } - break; - - default: - return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); - } - - AllocateTxId(); - } catch (const yexception&) { - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Invalid operation id"); - } - - Become(&TCancelOperationRPC::StateWait); - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExport::TEvCancelExportResponse, Handle); - hFunc(TEvImport::TEvCancelImportResponse, Handle); - hFunc(TEvIndexBuilder::TEvCancelResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - + case TOperationId::EXPORT: + case TOperationId::IMPORT: + case TOperationId::BUILD_INDEX: + if (!TryGetId(OperationId, RawOperationId)) { + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unable to extract operation id"); + } + break; + + default: + return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); + } + + AllocateTxId(); + } catch (const yexception&) { + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Invalid operation id"); + } + + Become(&TCancelOperationRPC::StateWait); + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExport::TEvCancelExportResponse, Handle); + hFunc(TEvImport::TEvCancelImportResponse, Handle); + hFunc(TEvIndexBuilder::TEvCancelResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + private: TOperationId OperationId; ui64 RawOperationId = 0; - -}; // TCancelOperationRPC - -void TGRpcRequestProxy::Handle(TEvCancelOperationRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TCancelOperationRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + +}; // TCancelOperationRPC + +void TGRpcRequestProxy::Handle(TEvCancelOperationRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TCancelOperationRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_common.h b/ydb/core/grpc_services/rpc_common.h index 7f0834184b2..075318037c9 100644 --- a/ydb/core/grpc_services/rpc_common.h +++ b/ydb/core/grpc_services/rpc_common.h @@ -32,18 +32,18 @@ inline void SetDatabase(TEv& ev, const IRequestCtx& ctx) { } inline void SetDatabase(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { - // Empty database in case of absent header - ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); -} - + // Empty database in case of absent header + ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); +} + inline void SetDatabase(TEvTxUserProxy::TEvNavigate* ev, const IRequestCtx& ctx) { - // Empty database in case of absent header - ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); -} - + // Empty database in case of absent header + ev->Record.SetDatabaseName(CanonizePath(ctx.GetDatabaseName().GetOrElse(""))); +} + inline void SetRequestType(TEvTxUserProxy::TEvProposeTransaction* ev, const IRequestCtx& ctx) { - ev->Record.SetRequestType(ctx.GetRequestType().GetOrElse("")); -} - + ev->Record.SetRequestType(ctx.GetRequestType().GetOrElse("")); +} + } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_copy_table.cpp b/ydb/core/grpc_services/rpc_copy_table.cpp index 90a18f13999..74a0c9d9a81 100644 --- a/ydb/core/grpc_services/rpc_copy_table.cpp +++ b/ydb/core/grpc_services/rpc_copy_table.cpp @@ -38,7 +38,7 @@ private: const auto& workingDir = destinationPathPair.first; const auto& name = destinationPathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_copy_tables.cpp b/ydb/core/grpc_services/rpc_copy_tables.cpp index f4e4a126fa2..1f5fd281a51 100644 --- a/ydb/core/grpc_services/rpc_copy_tables.cpp +++ b/ydb/core/grpc_services/rpc_copy_tables.cpp @@ -26,7 +26,7 @@ public: private: void SendProposeRequest(const TActorContext &ctx) { - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables); diff --git a/ydb/core/grpc_services/rpc_create_coordination_node.cpp b/ydb/core/grpc_services/rpc_create_coordination_node.cpp index 8cf3f18fab1..11c838592ca 100644 --- a/ydb/core/grpc_services/rpc_create_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_create_coordination_node.cpp @@ -29,7 +29,7 @@ private: const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { - pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); + pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); } catch (const std::exception& ex) { Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); @@ -38,7 +38,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_create_table.cpp b/ydb/core/grpc_services/rpc_create_table.cpp index 5815e01b27f..302dfde57a8 100644 --- a/ydb/core/grpc_services/rpc_create_table.cpp +++ b/ydb/core/grpc_services/rpc_create_table.cpp @@ -102,7 +102,7 @@ private: const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { - pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); + pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); } catch (const std::exception& ex) { Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return Reply(StatusIds::BAD_REQUEST, ctx); @@ -122,7 +122,7 @@ private: return Reply(StatusIds::BAD_REQUEST, ctx); } - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); @@ -137,21 +137,21 @@ private: tableDesc->SetName(name); - StatusIds::StatusCode code = StatusIds::SUCCESS; - TString error; + StatusIds::StatusCode code = StatusIds::SUCCESS; + TString error; if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); } - tableDesc->MutableKeyColumnNames()->CopyFrom(req->primary_key()); + tableDesc->MutableKeyColumnNames()->CopyFrom(req->primary_key()); - if (!FillIndexDescription(*modifyScheme->MutableCreateIndexedTable(), *req, code, error)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); + if (!FillIndexDescription(*modifyScheme->MutableCreateIndexedTable(), *req, code, error)) { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); } bool tableProfileSet = false; @@ -161,7 +161,7 @@ private: || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() || profile.has_caching_policy(); } - + if (!Profiles.ApplyTableProfile(req->profile(), *tableDesc, code, error)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); @@ -199,15 +199,15 @@ private: return Reply(code, issues, ctx); } - // Attributes - for (auto [key, value] : req->attributes()) { - auto& attr = *modifyScheme->MutableAlterUserAttributes()->AddUserAttributes(); - attr.SetKey(key); - attr.SetValue(value); - } - + // Attributes + for (auto [key, value] : req->attributes()) { + auto& attr = *modifyScheme->MutableAlterUserAttributes()->AddUserAttributes(); + attr.SetKey(key); + attr.SetValue(value); + } + TList<TString> warnings; - if (!FillCreateTableSettingsDesc(*tableDesc, *req, Profiles, code, error, warnings)) { + if (!FillCreateTableSettingsDesc(*tableDesc, *req, Profiles, code, error, warnings)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); return Reply(code, issues, ctx); diff --git a/ydb/core/grpc_services/rpc_deferrable.h b/ydb/core/grpc_services/rpc_deferrable.h index 3d943c12c06..644e7b84c40 100644 --- a/ydb/core/grpc_services/rpc_deferrable.h +++ b/ydb/core/grpc_services/rpc_deferrable.h @@ -1,7 +1,7 @@ #pragma once - + #include "defs.h" -#include "grpc_request_proxy.h" +#include "grpc_request_proxy.h" #include "rpc_common.h" #include <ydb/core/tx/tx_proxy/proxy.h> @@ -11,7 +11,7 @@ #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <ydb/core/actorlib_impl/long_timer.h> #include <library/cpp/actors/core/actor_bootstrapped.h> diff --git a/ydb/core/grpc_services/rpc_describe_coordination_node.cpp b/ydb/core/grpc_services/rpc_describe_coordination_node.cpp index 27db98febfd..04c16bf32e8 100644 --- a/ydb/core/grpc_services/rpc_describe_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_describe_coordination_node.cpp @@ -85,7 +85,7 @@ private: std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); SetAuthToken(navigateRequest, *Request_); - SetDatabase(navigateRequest.get(), *Request_); + SetDatabase(navigateRequest.get(), *Request_); NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath(); record->SetPath(req->path()); diff --git a/ydb/core/grpc_services/rpc_describe_path.cpp b/ydb/core/grpc_services/rpc_describe_path.cpp index ac7b2ffed16..7960f50779b 100644 --- a/ydb/core/grpc_services/rpc_describe_path.cpp +++ b/ydb/core/grpc_services/rpc_describe_path.cpp @@ -19,9 +19,9 @@ using TEvListDirectoryRequest = TGrpcRequestOperationCall<Ydb::Scheme::ListDirec using TEvDescribePathRequest = TGrpcRequestOperationCall<Ydb::Scheme::DescribePathRequest, Ydb::Scheme::DescribePathResponse>; -template <typename TDerived, typename TRequest, typename TResult, bool ListChildren = false> -class TBaseDescribe : public TRpcSchemeRequestActor<TDerived, TRequest> { - using TBase = TRpcSchemeRequestActor<TDerived, TRequest>; +template <typename TDerived, typename TRequest, typename TResult, bool ListChildren = false> +class TBaseDescribe : public TRpcSchemeRequestActor<TDerived, TRequest> { + using TBase = TRpcSchemeRequestActor<TDerived, TRequest>; public: TBaseDescribe(IRequestOpCtx* msg) @@ -31,22 +31,22 @@ public: TBase::Bootstrap(ctx); SendProposeRequest(ctx); - this->Become(&TDerived::StateWork); + this->Become(&TDerived::StateWork); } private: - void SendProposeRequest(const TActorContext &ctx) { - const auto req = this->GetProtoRequest(); - - std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); - SetAuthToken(navigateRequest, *this->Request_); - SetDatabase(navigateRequest.get(), *this->Request_); + void SendProposeRequest(const TActorContext &ctx) { + const auto req = this->GetProtoRequest(); + + std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); + SetAuthToken(navigateRequest, *this->Request_); + SetDatabase(navigateRequest.get(), *this->Request_); NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath(); - record->SetPath(req->path()); - - ctx.Send(MakeTxProxyID(), navigateRequest.release()); - } - + record->SetPath(req->path()); + + ctx.Send(MakeTxProxyID(), navigateRequest.release()); + } + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); @@ -57,55 +57,55 @@ private: void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->GetRecord(); const auto status = record.GetStatus(); - + if (record.HasReason()) { auto issue = NYql::TIssue(record.GetReason()); - this->Request_->RaiseIssue(issue); + this->Request_->RaiseIssue(issue); } - - TResult result; + + TResult result; switch (status) { case NKikimrScheme::StatusSuccess: { const auto& pathDescription = record.GetPathDescription(); - ConvertDirectoryEntry(pathDescription, result.mutable_self(), true); - if constexpr (ListChildren) { - for (const auto& child : pathDescription.GetChildren()) { - ConvertDirectoryEntry(child, result.add_children(), false); - } - } - return this->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); + ConvertDirectoryEntry(pathDescription, result.mutable_self(), true); + if constexpr (ListChildren) { + for (const auto& child : pathDescription.GetChildren()) { + ConvertDirectoryEntry(child, result.add_children(), false); + } + } + return this->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); } case NKikimrScheme::StatusPathDoesNotExist: case NKikimrScheme::StatusSchemeError: { - return this->Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); } case NKikimrScheme::StatusAccessDenied: { - return this->Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); + return this->Reply(Ydb::StatusIds::UNAUTHORIZED, ctx); } case NKikimrScheme::StatusNotAvailable: { - return this->Reply(Ydb::StatusIds::UNAVAILABLE, ctx); + return this->Reply(Ydb::StatusIds::UNAVAILABLE, ctx); } default: { - return this->Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); + return this->Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); } } } -}; +}; -class TListDirectoryRPC : public TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true> { -public: - using TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true>::TBaseDescribe; -}; +class TListDirectoryRPC : public TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true> { +public: + using TBaseDescribe<TListDirectoryRPC, TEvListDirectoryRequest, Ydb::Scheme::ListDirectoryResult, true>::TBaseDescribe; +}; -class TDescribePathRPC : public TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult> { -public: - using TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult>::TBaseDescribe; +class TDescribePathRPC : public TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult> { +public: + using TBaseDescribe<TDescribePathRPC, TEvDescribePathRequest, Ydb::Scheme::DescribePathResult>::TBaseDescribe; }; void DoListDirectoryRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TListDirectoryRPC(p.release())); -} - +} + void DoDescribePathRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TDescribePathRPC(p.release())); } diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp index b19d9125bd2..261f10d4676 100644 --- a/ydb/core/grpc_services/rpc_describe_table.cpp +++ b/ydb/core/grpc_services/rpc_describe_table.cpp @@ -82,7 +82,7 @@ private: FillStorageSettings(describeTableResult, tableDescription); FillColumnFamilies(describeTableResult, tableDescription); - FillAttributes(describeTableResult, pathDescription); + FillAttributes(describeTableResult, pathDescription); FillPartitioningSettings(describeTableResult, tableDescription); FillKeyBloomFilter(describeTableResult, tableDescription); FillReadReplicasSettings(describeTableResult, tableDescription); @@ -114,7 +114,7 @@ private: std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate()); SetAuthToken(navigateRequest, *Request_); - SetDatabase(navigateRequest.get(), *Request_); + SetDatabase(navigateRequest.get(), *Request_); NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath(); record->SetPath(req->path()); if (req->include_shard_key_bounds()) { diff --git a/ydb/core/grpc_services/rpc_discovery.cpp b/ydb/core/grpc_services/rpc_discovery.cpp index 743c819534c..75550f92acf 100644 --- a/ydb/core/grpc_services/rpc_discovery.cpp +++ b/ydb/core/grpc_services/rpc_discovery.cpp @@ -47,12 +47,12 @@ namespace NDiscoveryPrivate { THashMap<TString, THolder<TEvStateStorage::TEvBoardInfo>> OldInfo; THashMap<TString, THolder<TEvStateStorage::TEvBoardInfo>> NewInfo; - struct TWaiter { + struct TWaiter { TActorId ActorId; - ui64 Cookie; - }; - - THashMap<TString, TVector<TWaiter>> Requested; + ui64 Cookie; + }; + + THashMap<TString, TVector<TWaiter>> Requested; bool Scheduled; void Handle(TEvStateStorage::TEvBoardInfo::TPtr &ev) { @@ -62,7 +62,7 @@ namespace NDiscoveryPrivate { auto vecIt = Requested.find(path); if (vecIt != Requested.end()) { for (auto &x : vecIt->second) - Send(x.ActorId, new TEvStateStorage::TEvBoardInfo(*msg), 0, x.Cookie); + Send(x.ActorId, new TEvStateStorage::TEvBoardInfo(*msg), 0, x.Cookie); Requested.erase(vecIt); } @@ -89,12 +89,12 @@ namespace NDiscoveryPrivate { void Handle(TEvPrivate::TEvRequest::TPtr &ev) { auto *msg = ev->Get(); if (auto *x = OldInfo.FindPtr(msg->Database)) { - Send(ev->Sender, new TEvStateStorage::TEvBoardInfo(**x), 0, ev->Cookie); + Send(ev->Sender, new TEvStateStorage::TEvBoardInfo(**x), 0, ev->Cookie); return; } if (auto *x = NewInfo.FindPtr(msg->Database)) { - Send(ev->Sender, new TEvStateStorage::TEvBoardInfo(**x), 0, ev->Cookie); + Send(ev->Sender, new TEvStateStorage::TEvBoardInfo(**x), 0, ev->Cookie); return; } @@ -103,7 +103,7 @@ namespace NDiscoveryPrivate { Register(CreateBoardLookupActor(msg->Database, SelfId(), msg->StateStorageId, EBoardLookupMode::Second, false, false)); } - rqstd.push_back({ev->Sender, ev->Cookie}); + rqstd.push_back({ev->Sender, ev->Cookie}); } public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -135,10 +135,10 @@ class TListEndpointsRPC : public TActorBootstrapped<TListEndpointsRPC> { THolder<TEvStateStorage::TEvBoardInfo> LookupResponse; THolder<TEvInterconnect::TEvNodeInfo> NameserviceResponse; THolder<TEvTxProxySchemeCache::TEvNavigateKeySetResult> SchemeCacheResponse; - - bool ResolveResources = false; - ui64 LookupCookie = 0; - + + bool ResolveResources = false; + ui64 LookupCookie = 0; + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::GRPC_REQ; @@ -151,14 +151,14 @@ public: void Bootstrap() { // request endpoints - Lookup(Request->GetProtoRequest()->database()); + Lookup(Request->GetProtoRequest()->database()); // request self node info Send(GetNameserviceActorId(), new TEvInterconnect::TEvGetNode(SelfId().NodeId())); // request path info if (RequestScheme) { - Navigate(Request->GetProtoRequest()->database()); + Navigate(Request->GetProtoRequest()->database()); } Become(&TThis::StateWait); @@ -175,10 +175,10 @@ public: } void Handle(TEvStateStorage::TEvBoardInfo::TPtr &ev) { - if (ev->Cookie != LookupCookie) { - return; - } - + if (ev->Cookie != LookupCookie) { + return; + } + LookupResponse = THolder<TEvStateStorage::TEvBoardInfo>(ev->Release().Release()); TryReplyAndDie(); @@ -201,7 +201,7 @@ public: LOG_TRACE_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, "TListEndpointsRPC: handle TEvNavigateKeySetResult" - << ", entry: " << entry.ToString()); + << ", entry: " << entry.ToString()); if (navigate->ErrorCount > 0) { switch (entry.Status) { @@ -218,7 +218,7 @@ public: { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, "TListEndpointsRPC: GENERIC_RESOLVE_ERROR" - << ", entry: " << entry.ToString()); + << ", entry: " << entry.ToString()); auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, "Database resolve failed with no certain result"); google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; @@ -229,31 +229,31 @@ public: } } - if (!entry.DomainInfo) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, - "TListEndpointsRPC: GENERIC_RESOLVE_ERROR (empty domain info)" - << ", entry: " << entry.ToString()); - - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, "Database resolve failed with no certain result"); - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; - NYql::IssueToMessage(issue, issueMessages.Add()); - Request->SendResult(Ydb::StatusIds::UNAVAILABLE, issueMessages); - return PassAway(); - } - - auto info = entry.DomainInfo; - if (info->DomainKey != info->ResourcesDomainKey) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, - "TListEndpointsRPC: domain key differs from resources domain key" - << ", domain key: " << info->DomainKey - << ", resources domain key: " << info->ResourcesDomainKey); - - Navigate(info->ResourcesDomainKey); - ResolveResources = true; - } else if (ResolveResources) { - Lookup(CanonizePath(entry.Path)); - } - + if (!entry.DomainInfo) { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, + "TListEndpointsRPC: GENERIC_RESOLVE_ERROR (empty domain info)" + << ", entry: " << entry.ToString()); + + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, "Database resolve failed with no certain result"); + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; + NYql::IssueToMessage(issue, issueMessages.Add()); + Request->SendResult(Ydb::StatusIds::UNAVAILABLE, issueMessages); + return PassAway(); + } + + auto info = entry.DomainInfo; + if (info->DomainKey != info->ResourcesDomainKey) { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, + "TListEndpointsRPC: domain key differs from resources domain key" + << ", domain key: " << info->DomainKey + << ", resources domain key: " << info->ResourcesDomainKey); + + Navigate(info->ResourcesDomainKey); + ResolveResources = true; + } else if (ResolveResources) { + Lookup(CanonizePath(entry.Path)); + } + TryReplyAndDie(); } @@ -414,66 +414,66 @@ public: } return true; } - - void Lookup(const TString& db) { - TVector<TString> path = NKikimr::SplitPath(db); - auto domainName = path ? path[0] : TString(); - auto *appdata = AppData(); - auto *domainInfo = appdata->DomainsInfo->GetDomainByName(domainName); - if (!domainInfo) { - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DATABASE_NOT_EXIST, "Database " + domainName + " not exists"); - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; - NYql::IssueToMessage(issue, issueMessages.Add()); - Request->SendResult(Ydb::StatusIds::BAD_REQUEST, issueMessages); - return PassAway(); - } - - TString database; - for (auto &x : path) { - if (x.size() > 4100) { - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::KEY_PARSE_ERROR, "Requested database name too long"); - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; - NYql::IssueToMessage(issue, issueMessages.Add()); - Request->SendResult(Ydb::StatusIds::BAD_REQUEST, issueMessages); - return PassAway(); - } - database.append("/").append(x); - } - - // request endpoints - auto stateStorageGroupId = domainInfo->DefaultStateStorageGroup; - auto reqPath = MakeEndpointsBoardPath(database); - - Send(CacheId, new NDiscoveryPrivate::TEvPrivate::TEvRequest(reqPath, stateStorageGroupId), 0, ++LookupCookie); - LookupResponse.Reset(); - } - - void FillNavigateKey(const TString& path, NSchemeCache::TSchemeCacheNavigate::TEntry& entry) { - entry.Path = NKikimr::SplitPath(path); - entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByPath; - } - - void FillNavigateKey(const TPathId& pathId, NSchemeCache::TSchemeCacheNavigate::TEntry& entry) { - entry.TableId = TTableId(pathId.OwnerId, pathId.LocalPathId); - entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; - } - - template <typename T> - void Navigate(const T& id) { - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, - "TListEndpointsRPC: make TEvNavigateKeySet request" - << ", path: " << id); - - auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - - request->ResultSet.emplace_back(); - FillNavigateKey(id, request->ResultSet.back()); - request->ResultSet.back().Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - request->ResultSet.back().RedirectRequired = false; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), IEventHandle::FlagTrackDelivery); - SchemeCacheResponse.Reset(); - } + + void Lookup(const TString& db) { + TVector<TString> path = NKikimr::SplitPath(db); + auto domainName = path ? path[0] : TString(); + auto *appdata = AppData(); + auto *domainInfo = appdata->DomainsInfo->GetDomainByName(domainName); + if (!domainInfo) { + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DATABASE_NOT_EXIST, "Database " + domainName + " not exists"); + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; + NYql::IssueToMessage(issue, issueMessages.Add()); + Request->SendResult(Ydb::StatusIds::BAD_REQUEST, issueMessages); + return PassAway(); + } + + TString database; + for (auto &x : path) { + if (x.size() > 4100) { + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::KEY_PARSE_ERROR, "Requested database name too long"); + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issueMessages; + NYql::IssueToMessage(issue, issueMessages.Add()); + Request->SendResult(Ydb::StatusIds::BAD_REQUEST, issueMessages); + return PassAway(); + } + database.append("/").append(x); + } + + // request endpoints + auto stateStorageGroupId = domainInfo->DefaultStateStorageGroup; + auto reqPath = MakeEndpointsBoardPath(database); + + Send(CacheId, new NDiscoveryPrivate::TEvPrivate::TEvRequest(reqPath, stateStorageGroupId), 0, ++LookupCookie); + LookupResponse.Reset(); + } + + void FillNavigateKey(const TString& path, NSchemeCache::TSchemeCacheNavigate::TEntry& entry) { + entry.Path = NKikimr::SplitPath(path); + entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByPath; + } + + void FillNavigateKey(const TPathId& pathId, NSchemeCache::TSchemeCacheNavigate::TEntry& entry) { + entry.TableId = TTableId(pathId.OwnerId, pathId.LocalPathId); + entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; + } + + template <typename T> + void Navigate(const T& id) { + LOG_TRACE_S(*TlsActivationContext, NKikimrServices::GRPC_PROXY, + "TListEndpointsRPC: make TEvNavigateKeySet request" + << ", path: " << id); + + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + + request->ResultSet.emplace_back(); + FillNavigateKey(id, request->ResultSet.back()); + request->ResultSet.back().Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + request->ResultSet.back().RedirectRequired = false; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), IEventHandle::FlagTrackDelivery); + SchemeCacheResponse.Reset(); + } }; void TGRpcRequestProxy::Handle(TEvListEndpointsRequest::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/grpc_services/rpc_drop_coordination_node.cpp b/ydb/core/grpc_services/rpc_drop_coordination_node.cpp index 8629e371a4d..3632ec58723 100644 --- a/ydb/core/grpc_services/rpc_drop_coordination_node.cpp +++ b/ydb/core/grpc_services/rpc_drop_coordination_node.cpp @@ -38,7 +38,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_drop_table.cpp b/ydb/core/grpc_services/rpc_drop_table.cpp index 06a580af09c..27899a9d773 100644 --- a/ydb/core/grpc_services/rpc_drop_table.cpp +++ b/ydb/core/grpc_services/rpc_drop_table.cpp @@ -38,7 +38,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp index 6aa3a8def39..317a3488ce0 100644 --- a/ydb/core/grpc_services/rpc_execute_data_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp @@ -15,7 +15,7 @@ namespace NKikimr { namespace NGRpcService { using namespace NActors; -using namespace NOperationId; +using namespace NOperationId; using namespace Ydb; using namespace Ydb::Table; using namespace NKqp; diff --git a/ydb/core/grpc_services/rpc_export.cpp b/ydb/core/grpc_services/rpc_export.cpp index 53e0b83ffc7..f0fa7be3dfe 100644 --- a/ydb/core/grpc_services/rpc_export.cpp +++ b/ydb/core/grpc_services/rpc_export.cpp @@ -1,222 +1,222 @@ -#include "grpc_request_proxy.h" -#include "rpc_export_base.h" -#include "rpc_calls.h" -#include "rpc_operation_request_base.h" - +#include "grpc_request_proxy.h" +#include "rpc_export_base.h" +#include "rpc_calls.h" +#include "rpc_operation_request_base.h" + #include <ydb/core/tx/schemeshard/schemeshard_export.h> - + #include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/ptr.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +#include <util/generic/ptr.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NSchemeShard; -using namespace NKikimrIssues; -using namespace Ydb; - -template <typename TDerived, typename TEvRequest> -class TExportRPC: public TRpcOperationRequestActor<TDerived, TEvRequest, true>, public TExportConv { - TStringBuf GetLogPrefix() const override { - return "[CreateExport]"; - } - - IEventBase* MakeRequest() override { - const auto& request = *this->Request->GetProtoRequest(); - - auto ev = MakeHolder<TEvExport::TEvCreateExportRequest>(); - ev->Record.SetTxId(this->TxId); - ev->Record.SetDatabaseName(this->DatabaseName); - if (this->UserToken) { - ev->Record.SetUserSID(this->UserToken->GetUserSID()); - } - - auto& createExport = *ev->Record.MutableRequest(); - createExport.MutableOperationParams()->CopyFrom(request.operation_params()); - if (std::is_same_v<TEvRequest, TEvExportToYtRequest>) { - createExport.MutableExportToYtSettings()->CopyFrom(request.settings()); - } else if (std::is_same_v<TEvRequest, TEvExportToS3Request>) { - createExport.MutableExportToS3Settings()->CopyFrom(request.settings()); - } - - return ev.Release(); - } - - template <typename TProtoSettings> - static void ExtractPaths(TVector<TString>& paths, const TProtoSettings& settings) { - paths.reserve(settings.items_size() + 1); - for (const auto& item : settings.items()) { - paths.emplace_back(item.source_path()); - } - } - - TVector<TString> ExtractPaths() { - TVector<TString> paths; - - paths.emplace_back(this->DatabaseName); // first entry is database - ExtractPaths(paths, this->Request->GetProtoRequest()->settings()); - - return paths; - } - - void ResolvePaths(const TVector<TString>& paths) { - Y_VERIFY(!paths.empty()); - - auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - request->DatabaseName = this->DatabaseName; - - for (const auto& path : paths) { - auto& entry = request->ResultSet.emplace_back(); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - entry.Path = NKikimr::SplitPath(path); - if (entry.Path.empty()) { - return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR, "Cannot resolve empty path"); - } - } - - this->Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - this->Become(&TDerived::StateResolvePaths); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& request = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (request->ResultSet.empty()) { - return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - if (request->ErrorCount > 0) { - for (const auto& entry : request->ResultSet) { - if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - continue; - } - - StatusIds::StatusCode status; - TIssuesIds::EIssueCode code; - - switch (entry.Status) { - case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: - status = StatusIds::SCHEME_ERROR; - code = TIssuesIds::PATH_NOT_EXIST; - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: - case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: - status = StatusIds::UNAVAILABLE; - code = TIssuesIds::RESOLVE_LOOKUP_ERROR; - break; - default: - status = StatusIds::SCHEME_ERROR; - code = TIssuesIds::GENERIC_RESOLVE_ERROR; - break; - } - - return this->Reply(status, code, TStringBuilder() << "Cannot resolve path" - << ": path# " << CanonizePath(entry.Path) - << ", status# " << entry.Status); - } - } - - TString error; - - if (this->UserToken) { - bool isDatabase = true; // first entry is database - - for (const auto& entry : request->ResultSet) { - const ui32 access = isDatabase ? NACLib::GenericRead | NACLib::GenericWrite : NACLib::SelectRow; - if (!this->CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, access)) { - return; - } - - isDatabase = false; - } - } - - NSchemeCache::TDomainInfo::TPtr domainInfo; - for (const auto& entry : request->ResultSet) { - if (!entry.DomainInfo) { - LOG_E("Got empty domain info"); - return this->Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - if (!domainInfo) { - domainInfo = entry.DomainInfo; - continue; - } - - if (domainInfo->DomainKey != entry.DomainInfo->DomainKey) { - return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DOMAIN_LOCALITY_ERROR, - TStringBuilder() << "Failed locality check" - << ": expected# " << domainInfo->DomainKey - << ", actual# " << entry.DomainInfo->DomainKey); - } - } - - this->AllocateTxId(); - this->Become(&TDerived::StateWait); - } - - void Handle(TEvExport::TEvCreateExportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvCreateExportResponse" - << ": record# " << record.ShortDebugString()); - - this->Reply(TExportConv::ToOperation(record.GetEntry())); - } - -public: - using TRpcOperationRequestActor<TDerived, TEvRequest, true>::TRpcOperationRequestActor; - - void Bootstrap(const TActorContext&) { - const auto& request = *this->Request->GetProtoRequest(); - - if (request.settings().items().empty()) { - return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Items are not set"); - } - - ResolvePaths(ExtractPaths()); - } - - STATEFN(StateResolvePaths) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - } - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExport::TEvCreateExportResponse, Handle); - default: - return this->StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -}; // TExportRPC - -class TExportToYtRPC: public TExportRPC<TExportToYtRPC, TEvExportToYtRequest> { -public: - using TExportRPC::TExportRPC; -}; - -class TExportToS3RPC: public TExportRPC<TExportToS3RPC, TEvExportToS3Request> { -public: - using TExportRPC::TExportRPC; -}; - -void TGRpcRequestProxy::Handle(TEvExportToYtRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TExportToYtRPC(ev->Release().Release())); -} - -void TGRpcRequestProxy::Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TExportToS3RPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr +using namespace NKikimrIssues; +using namespace Ydb; + +template <typename TDerived, typename TEvRequest> +class TExportRPC: public TRpcOperationRequestActor<TDerived, TEvRequest, true>, public TExportConv { + TStringBuf GetLogPrefix() const override { + return "[CreateExport]"; + } + + IEventBase* MakeRequest() override { + const auto& request = *this->Request->GetProtoRequest(); + + auto ev = MakeHolder<TEvExport::TEvCreateExportRequest>(); + ev->Record.SetTxId(this->TxId); + ev->Record.SetDatabaseName(this->DatabaseName); + if (this->UserToken) { + ev->Record.SetUserSID(this->UserToken->GetUserSID()); + } + + auto& createExport = *ev->Record.MutableRequest(); + createExport.MutableOperationParams()->CopyFrom(request.operation_params()); + if (std::is_same_v<TEvRequest, TEvExportToYtRequest>) { + createExport.MutableExportToYtSettings()->CopyFrom(request.settings()); + } else if (std::is_same_v<TEvRequest, TEvExportToS3Request>) { + createExport.MutableExportToS3Settings()->CopyFrom(request.settings()); + } + + return ev.Release(); + } + + template <typename TProtoSettings> + static void ExtractPaths(TVector<TString>& paths, const TProtoSettings& settings) { + paths.reserve(settings.items_size() + 1); + for (const auto& item : settings.items()) { + paths.emplace_back(item.source_path()); + } + } + + TVector<TString> ExtractPaths() { + TVector<TString> paths; + + paths.emplace_back(this->DatabaseName); // first entry is database + ExtractPaths(paths, this->Request->GetProtoRequest()->settings()); + + return paths; + } + + void ResolvePaths(const TVector<TString>& paths) { + Y_VERIFY(!paths.empty()); + + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + request->DatabaseName = this->DatabaseName; + + for (const auto& path : paths) { + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = NKikimr::SplitPath(path); + if (entry.Path.empty()) { + return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR, "Cannot resolve empty path"); + } + } + + this->Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + this->Become(&TDerived::StateResolvePaths); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (request->ResultSet.empty()) { + return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + if (request->ErrorCount > 0) { + for (const auto& entry : request->ResultSet) { + if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { + continue; + } + + StatusIds::StatusCode status; + TIssuesIds::EIssueCode code; + + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + status = StatusIds::SCHEME_ERROR; + code = TIssuesIds::PATH_NOT_EXIST; + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + status = StatusIds::UNAVAILABLE; + code = TIssuesIds::RESOLVE_LOOKUP_ERROR; + break; + default: + status = StatusIds::SCHEME_ERROR; + code = TIssuesIds::GENERIC_RESOLVE_ERROR; + break; + } + + return this->Reply(status, code, TStringBuilder() << "Cannot resolve path" + << ": path# " << CanonizePath(entry.Path) + << ", status# " << entry.Status); + } + } + + TString error; + + if (this->UserToken) { + bool isDatabase = true; // first entry is database + + for (const auto& entry : request->ResultSet) { + const ui32 access = isDatabase ? NACLib::GenericRead | NACLib::GenericWrite : NACLib::SelectRow; + if (!this->CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, access)) { + return; + } + + isDatabase = false; + } + } + + NSchemeCache::TDomainInfo::TPtr domainInfo; + for (const auto& entry : request->ResultSet) { + if (!entry.DomainInfo) { + LOG_E("Got empty domain info"); + return this->Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + if (!domainInfo) { + domainInfo = entry.DomainInfo; + continue; + } + + if (domainInfo->DomainKey != entry.DomainInfo->DomainKey) { + return this->Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DOMAIN_LOCALITY_ERROR, + TStringBuilder() << "Failed locality check" + << ": expected# " << domainInfo->DomainKey + << ", actual# " << entry.DomainInfo->DomainKey); + } + } + + this->AllocateTxId(); + this->Become(&TDerived::StateWait); + } + + void Handle(TEvExport::TEvCreateExportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvCreateExportResponse" + << ": record# " << record.ShortDebugString()); + + this->Reply(TExportConv::ToOperation(record.GetEntry())); + } + +public: + using TRpcOperationRequestActor<TDerived, TEvRequest, true>::TRpcOperationRequestActor; + + void Bootstrap(const TActorContext&) { + const auto& request = *this->Request->GetProtoRequest(); + + if (request.settings().items().empty()) { + return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Items are not set"); + } + + ResolvePaths(ExtractPaths()); + } + + STATEFN(StateResolvePaths) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + } + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExport::TEvCreateExportResponse, Handle); + default: + return this->StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + +}; // TExportRPC + +class TExportToYtRPC: public TExportRPC<TExportToYtRPC, TEvExportToYtRequest> { +public: + using TExportRPC::TExportRPC; +}; + +class TExportToS3RPC: public TExportRPC<TExportToS3RPC, TEvExportToS3Request> { +public: + using TExportRPC::TExportRPC; +}; + +void TGRpcRequestProxy::Handle(TEvExportToYtRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TExportToYtRPC(ev->Release().Release())); +} + +void TGRpcRequestProxy::Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TExportToS3RPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_export_base.h b/ydb/core/grpc_services/rpc_export_base.h index af2c7a55525..1004b93e98c 100644 --- a/ydb/core/grpc_services/rpc_export_base.h +++ b/ydb/core/grpc_services/rpc_export_base.h @@ -1,82 +1,82 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/export.pb.h> #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/lib/operation_id/operation_id.h> - -#include <util/string/cast.h> - -namespace NKikimr { -namespace NGRpcService { - -struct TExportConv { - static Ydb::TOperationId MakeOperationId(const ui64 id, NKikimrExport::TExport::SettingsCase kind) { - Ydb::TOperationId operationId; - operationId.SetKind(Ydb::TOperationId::EXPORT); - NOperationId::AddOptionalValue(operationId, "id", ToString(id)); - - switch (kind) { - case NKikimrExport::TExport::kExportToYtSettings: - NOperationId::AddOptionalValue(operationId, "kind", "yt"); - break; - case NKikimrExport::TExport::kExportToS3Settings: - NOperationId::AddOptionalValue(operationId, "kind", "s3"); - break; - default: - Y_VERIFY_DEBUG(false, "Unknown export kind"); - break; - } - - return operationId; - } - - static Ydb::Operations::Operation ToOperation(const NKikimrExport::TExport& exprt) { - Ydb::Operations::Operation operation; - - operation.set_id(NOperationId::ProtoToString(MakeOperationId(exprt.GetId(), exprt.GetSettingsCase()))); - operation.set_status(exprt.GetStatus()); - if (operation.status() == Ydb::StatusIds::SUCCESS) { - operation.set_ready(exprt.GetProgress() == Ydb::Export::ExportProgress::PROGRESS_DONE); - } else { - operation.set_ready(true); - } - if (exprt.IssuesSize()) { - operation.mutable_issues()->CopyFrom(exprt.GetIssues()); - } - - using namespace Ydb::Export; - switch (exprt.GetSettingsCase()) { - case NKikimrExport::TExport::kExportToYtSettings: - Fill<ExportToYtMetadata, ExportToYtResult>(operation, exprt, exprt.GetExportToYtSettings()); - break; - case NKikimrExport::TExport::kExportToS3Settings: - Fill<ExportToS3Metadata, ExportToS3Result>(operation, exprt, exprt.GetExportToS3Settings()); - break; - default: - Y_VERIFY_DEBUG(false, "Unknown export kind"); - break; - } - - return operation; - } - -private: - template <typename TMetadata, typename TResult, typename TSettings> - static void Fill( - Ydb::Operations::Operation& operation, - const NKikimrExport::TExport& exprt, - const TSettings& settings) { - TMetadata metadata; - metadata.mutable_settings()->CopyFrom(settings); - metadata.set_progress(exprt.GetProgress()); - metadata.mutable_items_progress()->CopyFrom(exprt.GetItemsProgress()); - operation.mutable_metadata()->PackFrom(metadata); - - TResult result; - operation.mutable_result()->PackFrom(result); - } - -}; // TExportConv - -} // namespace NGRpcService -} // namespace NKikimr + +#include <util/string/cast.h> + +namespace NKikimr { +namespace NGRpcService { + +struct TExportConv { + static Ydb::TOperationId MakeOperationId(const ui64 id, NKikimrExport::TExport::SettingsCase kind) { + Ydb::TOperationId operationId; + operationId.SetKind(Ydb::TOperationId::EXPORT); + NOperationId::AddOptionalValue(operationId, "id", ToString(id)); + + switch (kind) { + case NKikimrExport::TExport::kExportToYtSettings: + NOperationId::AddOptionalValue(operationId, "kind", "yt"); + break; + case NKikimrExport::TExport::kExportToS3Settings: + NOperationId::AddOptionalValue(operationId, "kind", "s3"); + break; + default: + Y_VERIFY_DEBUG(false, "Unknown export kind"); + break; + } + + return operationId; + } + + static Ydb::Operations::Operation ToOperation(const NKikimrExport::TExport& exprt) { + Ydb::Operations::Operation operation; + + operation.set_id(NOperationId::ProtoToString(MakeOperationId(exprt.GetId(), exprt.GetSettingsCase()))); + operation.set_status(exprt.GetStatus()); + if (operation.status() == Ydb::StatusIds::SUCCESS) { + operation.set_ready(exprt.GetProgress() == Ydb::Export::ExportProgress::PROGRESS_DONE); + } else { + operation.set_ready(true); + } + if (exprt.IssuesSize()) { + operation.mutable_issues()->CopyFrom(exprt.GetIssues()); + } + + using namespace Ydb::Export; + switch (exprt.GetSettingsCase()) { + case NKikimrExport::TExport::kExportToYtSettings: + Fill<ExportToYtMetadata, ExportToYtResult>(operation, exprt, exprt.GetExportToYtSettings()); + break; + case NKikimrExport::TExport::kExportToS3Settings: + Fill<ExportToS3Metadata, ExportToS3Result>(operation, exprt, exprt.GetExportToS3Settings()); + break; + default: + Y_VERIFY_DEBUG(false, "Unknown export kind"); + break; + } + + return operation; + } + +private: + template <typename TMetadata, typename TResult, typename TSettings> + static void Fill( + Ydb::Operations::Operation& operation, + const NKikimrExport::TExport& exprt, + const TSettings& settings) { + TMetadata metadata; + metadata.mutable_settings()->CopyFrom(settings); + metadata.set_progress(exprt.GetProgress()); + metadata.mutable_items_progress()->CopyFrom(exprt.GetItemsProgress()); + operation.mutable_metadata()->PackFrom(metadata); + + TResult result; + operation.mutable_result()->PackFrom(result); + } + +}; // TExportConv + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_forget_operation.cpp b/ydb/core/grpc_services/rpc_forget_operation.cpp index 64c61b5e3f9..8b250125cac 100644 --- a/ydb/core/grpc_services/rpc_forget_operation.cpp +++ b/ydb/core/grpc_services/rpc_forget_operation.cpp @@ -1,127 +1,127 @@ -#include "grpc_request_proxy.h" -#include "operation_helpers.h" -#include "rpc_calls.h" -#include "rpc_operation_request_base.h" - +#include "grpc_request_proxy.h" +#include "operation_helpers.h" +#include "rpc_calls.h" +#include "rpc_operation_request_base.h" + #include <ydb/core/tx/schemeshard/schemeshard_build_index.h> #include <ydb/core/tx/schemeshard/schemeshard_export.h> #include <ydb/core/tx/schemeshard/schemeshard_import.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NSchemeShard; -using namespace NKikimrIssues; -using namespace NOperationId; -using namespace Ydb; - -class TForgetOperationRPC: public TRpcOperationRequestActor<TForgetOperationRPC, TEvForgetOperationRequest> { - TStringBuf GetLogPrefix() const override { - switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - return "[ForgetExport]"; - case TOperationId::IMPORT: - return "[ForgetImport]"; - case TOperationId::BUILD_INDEX: - return "[ForgetIndexBuild]"; - default: - return "[Untagged]"; - } - } - - IEventBase* MakeRequest() override { - switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - return new TEvExport::TEvForgetExportRequest(TxId, DatabaseName, RawOperationId); - case TOperationId::IMPORT: - return new TEvImport::TEvForgetImportRequest(TxId, DatabaseName, RawOperationId); - case TOperationId::BUILD_INDEX: - return new TEvIndexBuilder::TEvForgetRequest(TxId, DatabaseName, RawOperationId); - default: - Y_FAIL("unreachable"); - } - } - - void Handle(TEvExport::TEvForgetExportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvForgetExportResponse" - << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); - } - - void Handle(TEvImport::TEvForgetImportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvImport::TEvForgetImportResponse" - << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); - } - - void Handle(TEvIndexBuilder::TEvForgetResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - LOG_D("Handle TEvIndexBuilder::TEvForgetResponse" - << ": record# " << record.ShortDebugString()); - - Reply(record.GetStatus(), record.GetIssues()); - } - -public: - using TRpcOperationRequestActor::TRpcOperationRequestActor; - - void Bootstrap() { - const TString& id = Request->GetProtoRequest()->id(); - - try { +using namespace NKikimrIssues; +using namespace NOperationId; +using namespace Ydb; + +class TForgetOperationRPC: public TRpcOperationRequestActor<TForgetOperationRPC, TEvForgetOperationRequest> { + TStringBuf GetLogPrefix() const override { + switch (OperationId.GetKind()) { + case TOperationId::EXPORT: + return "[ForgetExport]"; + case TOperationId::IMPORT: + return "[ForgetImport]"; + case TOperationId::BUILD_INDEX: + return "[ForgetIndexBuild]"; + default: + return "[Untagged]"; + } + } + + IEventBase* MakeRequest() override { + switch (OperationId.GetKind()) { + case TOperationId::EXPORT: + return new TEvExport::TEvForgetExportRequest(TxId, DatabaseName, RawOperationId); + case TOperationId::IMPORT: + return new TEvImport::TEvForgetImportRequest(TxId, DatabaseName, RawOperationId); + case TOperationId::BUILD_INDEX: + return new TEvIndexBuilder::TEvForgetRequest(TxId, DatabaseName, RawOperationId); + default: + Y_FAIL("unreachable"); + } + } + + void Handle(TEvExport::TEvForgetExportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvForgetExportResponse" + << ": record# " << record.ShortDebugString()); + + Reply(record.GetStatus(), record.GetIssues()); + } + + void Handle(TEvImport::TEvForgetImportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvImport::TEvForgetImportResponse" + << ": record# " << record.ShortDebugString()); + + Reply(record.GetStatus(), record.GetIssues()); + } + + void Handle(TEvIndexBuilder::TEvForgetResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + LOG_D("Handle TEvIndexBuilder::TEvForgetResponse" + << ": record# " << record.ShortDebugString()); + + Reply(record.GetStatus(), record.GetIssues()); + } + +public: + using TRpcOperationRequestActor::TRpcOperationRequestActor; + + void Bootstrap() { + const TString& id = Request->GetProtoRequest()->id(); + + try { OperationId = TOperationId(id); - + switch (OperationId.GetKind()) { - case TOperationId::EXPORT: - case TOperationId::IMPORT: - case TOperationId::BUILD_INDEX: - if (!TryGetId(OperationId, RawOperationId)) { - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unable to extract operation id"); - } - break; - - default: - return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); - } - - AllocateTxId(); - } catch (const yexception&) { - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Invalid operation id"); - } - - Become(&TForgetOperationRPC::StateWait); - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExport::TEvForgetExportResponse, Handle); - hFunc(TEvImport::TEvForgetImportResponse, Handle); - hFunc(TEvIndexBuilder::TEvForgetResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); + case TOperationId::EXPORT: + case TOperationId::IMPORT: + case TOperationId::BUILD_INDEX: + if (!TryGetId(OperationId, RawOperationId)) { + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unable to extract operation id"); + } + break; + + default: + return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); + } + + AllocateTxId(); + } catch (const yexception&) { + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Invalid operation id"); + } + + Become(&TForgetOperationRPC::StateWait); + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExport::TEvForgetExportResponse, Handle); + hFunc(TEvImport::TEvForgetImportResponse, Handle); + hFunc(TEvIndexBuilder::TEvForgetResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); } } private: - TOperationId OperationId; + TOperationId OperationId; ui64 RawOperationId = 0; - -}; // TForgetOperationRPC - -void TGRpcRequestProxy::Handle(TEvForgetOperationRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TForgetOperationRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + +}; // TForgetOperationRPC + +void TGRpcRequestProxy::Handle(TEvForgetOperationRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TForgetOperationRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_get_operation.cpp b/ydb/core/grpc_services/rpc_get_operation.cpp index ab66b13cc5f..6f7a1406863 100644 --- a/ydb/core/grpc_services/rpc_get_operation.cpp +++ b/ydb/core/grpc_services/rpc_get_operation.cpp @@ -1,8 +1,8 @@ #include "grpc_request_proxy.h" -#include "operation_helpers.h" -#include "rpc_export_base.h" -#include "rpc_import_base.h" -#include "rpc_operation_request_base.h" +#include "operation_helpers.h" +#include "rpc_export_base.h" +#include "rpc_import_base.h" +#include "rpc_operation_request_base.h" #include <google/protobuf/text_format.h> @@ -17,79 +17,79 @@ #include <ydb/public/lib/operation_id/operation_id.h> #include <library/cpp/actors/core/hfunc.h> - + #include <util/string/cast.h> namespace NKikimr { namespace NGRpcService { using namespace NActors; -using namespace NOperationId; +using namespace NOperationId; using namespace Ydb; -class TGetOperationRPC : public TRpcOperationRequestActor<TGetOperationRPC, TEvGetOperationRequest, true>, - public TExportConv { - - TStringBuf GetLogPrefix() const override { - switch (OperationId_.GetKind()) { - case TOperationId::EXPORT: - return "[GetExport]"; - case TOperationId::IMPORT: - return "[GetImport]"; - case TOperationId::BUILD_INDEX: - return "[GetIndexBuild]"; - default: - return "[Untagged]"; +class TGetOperationRPC : public TRpcOperationRequestActor<TGetOperationRPC, TEvGetOperationRequest, true>, + public TExportConv { + + TStringBuf GetLogPrefix() const override { + switch (OperationId_.GetKind()) { + case TOperationId::EXPORT: + return "[GetExport]"; + case TOperationId::IMPORT: + return "[GetImport]"; + case TOperationId::BUILD_INDEX: + return "[GetIndexBuild]"; + default: + return "[Untagged]"; } - } + } - IEventBase* MakeRequest() override { - switch (OperationId_.GetKind()) { - case TOperationId::EXPORT: + IEventBase* MakeRequest() override { + switch (OperationId_.GetKind()) { + case TOperationId::EXPORT: return new NSchemeShard::TEvExport::TEvGetExportRequest(DatabaseName, RawOperationId_); - case TOperationId::IMPORT: + case TOperationId::IMPORT: return new NSchemeShard::TEvImport::TEvGetImportRequest(DatabaseName, RawOperationId_); - case TOperationId::BUILD_INDEX: + case TOperationId::BUILD_INDEX: return new NSchemeShard::TEvIndexBuilder::TEvGetRequest(DatabaseName, RawOperationId_); - default: - Y_FAIL("unreachable"); - } - } - - void PassAway() override { + default: + Y_FAIL("unreachable"); + } + } + + void PassAway() override { if (PipeActorId_) { NTabletPipe::CloseClient(SelfId(), PipeActorId_); PipeActorId_ = TActorId(); } - TRpcOperationRequestActor::PassAway(); + TRpcOperationRequestActor::PassAway(); } public: - using TRpcOperationRequestActor::TRpcOperationRequestActor; + using TRpcOperationRequestActor::TRpcOperationRequestActor; void Bootstrap(const TActorContext &ctx) { - const auto req = Request->GetProtoRequest(); + const auto req = Request->GetProtoRequest(); try { OperationId_ = TOperationId(req->id()); - + switch (OperationId_.GetKind()) { - case TOperationId::CMS_REQUEST: + case TOperationId::CMS_REQUEST: SendCheckCmsOperation(ctx); - break; - case TOperationId::EXPORT: - case TOperationId::IMPORT: - case TOperationId::BUILD_INDEX: - if (!TryGetId(OperationId_, RawOperationId_)) { - return ReplyWithStatus(StatusIds::BAD_REQUEST); - } - ResolveDatabase(); - break; - default: + break; + case TOperationId::EXPORT: + case TOperationId::IMPORT: + case TOperationId::BUILD_INDEX: + if (!TryGetId(OperationId_, RawOperationId_)) { + return ReplyWithStatus(StatusIds::BAD_REQUEST); + } + ResolveDatabase(); + break; + default: SendNotifyTxCompletion(ctx); - break; - } + break; + } } catch (const yexception& ex) { return ReplyWithStatus(StatusIds::BAD_REQUEST); } @@ -107,7 +107,7 @@ public: HFunc(NSchemeShard::TEvIndexBuilder::TEvGetResponse, Handle); default: - return StateBase(ev, TlsActivationContext->AsActorContext()); + return StateBase(ev, TlsActivationContext->AsActorContext()); } } private: @@ -144,7 +144,7 @@ private: return ReplyWithStatus(StatusIds::BAD_REQUEST); } } catch (const yexception& ex) { - Request->RaiseIssue(NYql::ExceptionToIssue(ex)); + Request->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithStatus(StatusIds::BAD_REQUEST); } @@ -153,8 +153,8 @@ private: PipeActorId_ = ctx.ExecutorThread.RegisterActor(pipeActor); auto request = MakeHolder<NConsole::TEvConsole::TEvGetOperationRequest>(); - request->Record.MutableRequest()->set_id(Request->GetProtoRequest()->id()); - request->Record.SetUserToken(Request->GetInternalToken()); + request->Record.MutableRequest()->set_id(Request->GetProtoRequest()->id()); + request->Record.SetUserToken(Request->GetInternalToken()); NTabletPipe::SendData(ctx, PipeActorId_, request.Release()); } @@ -185,33 +185,33 @@ private: } void Handle(NSchemeShard::TEvExport::TEvGetExportResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvGetExportResponse" - << ": record# " << record.ShortDebugString()); - - TEvGetOperationRequest::TResponse resp; - *resp.mutable_operation() = TExportConv::ToOperation(record.GetEntry()); - Reply(resp, ctx); - } - + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvGetExportResponse" + << ": record# " << record.ShortDebugString()); + + TEvGetOperationRequest::TResponse resp; + *resp.mutable_operation() = TExportConv::ToOperation(record.GetEntry()); + Reply(resp, ctx); + } + void Handle(NSchemeShard::TEvImport::TEvGetImportResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record.GetResponse(); + const auto& record = ev->Get()->Record.GetResponse(); - LOG_D("Handle TEvImport::TEvGetImportResponse" - << ": record# " << record.ShortDebugString()); + LOG_D("Handle TEvImport::TEvGetImportResponse" + << ": record# " << record.ShortDebugString()); - TEvGetOperationRequest::TResponse resp; - *resp.mutable_operation() = TImportConv::ToOperation(record.GetEntry()); - Reply(resp, ctx); + TEvGetOperationRequest::TResponse resp; + *resp.mutable_operation() = TImportConv::ToOperation(record.GetEntry()); + Reply(resp, ctx); } void Handle(NSchemeShard::TEvIndexBuilder::TEvGetResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; - LOG_D("Handle TEvIndexBuilder::TEvGetResponse" - << ": record# " << record.ShortDebugString()); - + LOG_D("Handle TEvIndexBuilder::TEvGetResponse" + << ": record# " << record.ShortDebugString()); + if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { ReplyGetOperationResponse(true, ctx, record.GetStatus()); } else { @@ -227,32 +227,32 @@ private: const TActorContext &ctx) { TEvGetOperationRequest::TResponse resp; auto deferred = resp.mutable_operation(); - deferred->set_id(Request->GetProtoRequest()->id()); + deferred->set_id(Request->GetProtoRequest()->id()); deferred->set_ready(true); deferred->set_status(status); if (issues.size()) deferred->mutable_issues()->CopyFrom(issues); - Reply(resp, ctx); + Reply(resp, ctx); } void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, StatusIds::StatusCode status = StatusIds::SUCCESS) { TEvGetOperationRequest::TResponse resp; auto deferred = resp.mutable_operation(); - deferred->set_id(Request->GetProtoRequest()->id()); + deferred->set_id(Request->GetProtoRequest()->id()); deferred->set_ready(ready); if (ready) { deferred->set_status(status); } - Reply(resp, ctx); - } - + Reply(resp, ctx); + } + template<typename TMessage> void ReplyGetOperationResponse(bool ready, const TActorContext& ctx, const TMessage& metadata, StatusIds::StatusCode status = StatusIds::SUCCESS) { TEvGetOperationRequest::TResponse resp; auto deferred = resp.mutable_operation(); - deferred->set_id(Request->GetProtoRequest()->id()); + deferred->set_id(Request->GetProtoRequest()->id()); deferred->set_ready(ready); if (ready) { deferred->set_status(status); @@ -263,7 +263,7 @@ private: } - void Reply(const TEvGetOperationRequest::TResponse& response, const TActorContext& ctx) { + void Reply(const TEvGetOperationRequest::TResponse& response, const TActorContext& ctx) { TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), Request); this->Die(ctx); } diff --git a/ydb/core/grpc_services/rpc_import.cpp b/ydb/core/grpc_services/rpc_import.cpp index 94f90264c06..83eff924039 100644 --- a/ydb/core/grpc_services/rpc_import.cpp +++ b/ydb/core/grpc_services/rpc_import.cpp @@ -1,89 +1,89 @@ -#include "grpc_request_proxy.h" -#include "rpc_import_base.h" -#include "rpc_calls.h" -#include "rpc_operation_request_base.h" - +#include "grpc_request_proxy.h" +#include "rpc_import_base.h" +#include "rpc_calls.h" +#include "rpc_operation_request_base.h" + #include <ydb/core/tx/schemeshard/schemeshard_import.h> - -#include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/ptr.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +#include <library/cpp/actors/core/hfunc.h> + +#include <util/generic/ptr.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NSchemeShard; -using namespace NKikimrIssues; -using namespace Ydb; - -template <typename TDerived, typename TEvRequest> -class TImportRPC: public TRpcOperationRequestActor<TDerived, TEvRequest, true>, public TImportConv { - TStringBuf GetLogPrefix() const override { - return "[CreateImport]"; - } - - IEventBase* MakeRequest() override { - const auto& request = *this->Request->GetProtoRequest(); - - auto ev = MakeHolder<TEvImport::TEvCreateImportRequest>(); - ev->Record.SetTxId(this->TxId); - ev->Record.SetDatabaseName(this->DatabaseName); - if (this->UserToken) { - ev->Record.SetUserSID(this->UserToken->GetUserSID()); - } - - auto& createImport = *ev->Record.MutableRequest(); - createImport.MutableOperationParams()->CopyFrom(request.operation_params()); - if (std::is_same_v<TEvRequest, TEvImportFromS3Request>) { - createImport.MutableImportFromS3Settings()->CopyFrom(request.settings()); - } - - return ev.Release(); - } - - void Handle(TEvImport::TEvCreateImportResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvImport::TEvCreateImportResponse" - << ": record# " << record.ShortDebugString()); - - this->Reply(TImportConv::ToOperation(record.GetEntry())); - } - -public: - using TRpcOperationRequestActor<TDerived, TEvRequest, true>::TRpcOperationRequestActor; - - void Bootstrap(const TActorContext&) { - const auto& request = *this->Request->GetProtoRequest(); - - if (request.settings().items().empty()) { - return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Items are not set"); - } - - this->AllocateTxId(); - this->Become(&TDerived::StateWait); - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvImport::TEvCreateImportResponse, Handle); - default: - return this->StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -}; // TImportRPC - -class TImportFromS3RPC: public TImportRPC<TImportFromS3RPC, TEvImportFromS3Request> { -public: - using TImportRPC::TImportRPC; -}; - -void TGRpcRequestProxy::Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TImportFromS3RPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr +using namespace NKikimrIssues; +using namespace Ydb; + +template <typename TDerived, typename TEvRequest> +class TImportRPC: public TRpcOperationRequestActor<TDerived, TEvRequest, true>, public TImportConv { + TStringBuf GetLogPrefix() const override { + return "[CreateImport]"; + } + + IEventBase* MakeRequest() override { + const auto& request = *this->Request->GetProtoRequest(); + + auto ev = MakeHolder<TEvImport::TEvCreateImportRequest>(); + ev->Record.SetTxId(this->TxId); + ev->Record.SetDatabaseName(this->DatabaseName); + if (this->UserToken) { + ev->Record.SetUserSID(this->UserToken->GetUserSID()); + } + + auto& createImport = *ev->Record.MutableRequest(); + createImport.MutableOperationParams()->CopyFrom(request.operation_params()); + if (std::is_same_v<TEvRequest, TEvImportFromS3Request>) { + createImport.MutableImportFromS3Settings()->CopyFrom(request.settings()); + } + + return ev.Release(); + } + + void Handle(TEvImport::TEvCreateImportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvImport::TEvCreateImportResponse" + << ": record# " << record.ShortDebugString()); + + this->Reply(TImportConv::ToOperation(record.GetEntry())); + } + +public: + using TRpcOperationRequestActor<TDerived, TEvRequest, true>::TRpcOperationRequestActor; + + void Bootstrap(const TActorContext&) { + const auto& request = *this->Request->GetProtoRequest(); + + if (request.settings().items().empty()) { + return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Items are not set"); + } + + this->AllocateTxId(); + this->Become(&TDerived::StateWait); + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvImport::TEvCreateImportResponse, Handle); + default: + return this->StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + +}; // TImportRPC + +class TImportFromS3RPC: public TImportRPC<TImportFromS3RPC, TEvImportFromS3Request> { +public: + using TImportRPC::TImportRPC; +}; + +void TGRpcRequestProxy::Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TImportFromS3RPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_import_base.h b/ydb/core/grpc_services/rpc_import_base.h index c014d5bf8ff..f0d7d8985d0 100644 --- a/ydb/core/grpc_services/rpc_import_base.h +++ b/ydb/core/grpc_services/rpc_import_base.h @@ -1,76 +1,76 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/import.pb.h> #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/lib/operation_id/operation_id.h> - -#include <util/string/cast.h> - -namespace NKikimr { -namespace NGRpcService { - -struct TImportConv { - static Ydb::TOperationId MakeOperationId(const ui64 id, NKikimrImport::TImport::SettingsCase kind) { - Ydb::TOperationId operationId; - operationId.SetKind(Ydb::TOperationId::IMPORT); - NOperationId::AddOptionalValue(operationId, "id", ToString(id)); - - switch (kind) { - case NKikimrImport::TImport::kImportFromS3Settings: - NOperationId::AddOptionalValue(operationId, "kind", "s3"); - break; - default: - Y_VERIFY_DEBUG(false, "Unknown import kind"); - break; - } - - return operationId; - } - - static Ydb::Operations::Operation ToOperation(const NKikimrImport::TImport& import) { - Ydb::Operations::Operation operation; - - operation.set_id(NOperationId::ProtoToString(MakeOperationId(import.GetId(), import.GetSettingsCase()))); - operation.set_status(import.GetStatus()); - if (operation.status() == Ydb::StatusIds::SUCCESS) { - operation.set_ready(import.GetProgress() == Ydb::Import::ImportProgress::PROGRESS_DONE); - } else { - operation.set_ready(true); - } - if (import.IssuesSize()) { - operation.mutable_issues()->CopyFrom(import.GetIssues()); - } - - using namespace Ydb::Import; - switch (import.GetSettingsCase()) { - case NKikimrImport::TImport::kImportFromS3Settings: - Fill<ImportFromS3Metadata, ImportFromS3Result>(operation, import, import.GetImportFromS3Settings()); - break; - default: - Y_VERIFY_DEBUG(false, "Unknown import kind"); - break; - } - - return operation; - } - -private: - template <typename TMetadata, typename TResult, typename TSettings> - static void Fill( - Ydb::Operations::Operation& operation, - const NKikimrImport::TImport& import, - const TSettings& settings) { - TMetadata metadata; - metadata.mutable_settings()->CopyFrom(settings); - metadata.set_progress(import.GetProgress()); - metadata.mutable_items_progress()->CopyFrom(import.GetItemsProgress()); - operation.mutable_metadata()->PackFrom(metadata); - - TResult result; - operation.mutable_result()->PackFrom(result); - } - -}; // TImportConv - -} // namespace NGRpcService -} // namespace NKikimr + +#include <util/string/cast.h> + +namespace NKikimr { +namespace NGRpcService { + +struct TImportConv { + static Ydb::TOperationId MakeOperationId(const ui64 id, NKikimrImport::TImport::SettingsCase kind) { + Ydb::TOperationId operationId; + operationId.SetKind(Ydb::TOperationId::IMPORT); + NOperationId::AddOptionalValue(operationId, "id", ToString(id)); + + switch (kind) { + case NKikimrImport::TImport::kImportFromS3Settings: + NOperationId::AddOptionalValue(operationId, "kind", "s3"); + break; + default: + Y_VERIFY_DEBUG(false, "Unknown import kind"); + break; + } + + return operationId; + } + + static Ydb::Operations::Operation ToOperation(const NKikimrImport::TImport& import) { + Ydb::Operations::Operation operation; + + operation.set_id(NOperationId::ProtoToString(MakeOperationId(import.GetId(), import.GetSettingsCase()))); + operation.set_status(import.GetStatus()); + if (operation.status() == Ydb::StatusIds::SUCCESS) { + operation.set_ready(import.GetProgress() == Ydb::Import::ImportProgress::PROGRESS_DONE); + } else { + operation.set_ready(true); + } + if (import.IssuesSize()) { + operation.mutable_issues()->CopyFrom(import.GetIssues()); + } + + using namespace Ydb::Import; + switch (import.GetSettingsCase()) { + case NKikimrImport::TImport::kImportFromS3Settings: + Fill<ImportFromS3Metadata, ImportFromS3Result>(operation, import, import.GetImportFromS3Settings()); + break; + default: + Y_VERIFY_DEBUG(false, "Unknown import kind"); + break; + } + + return operation; + } + +private: + template <typename TMetadata, typename TResult, typename TSettings> + static void Fill( + Ydb::Operations::Operation& operation, + const NKikimrImport::TImport& import, + const TSettings& settings) { + TMetadata metadata; + metadata.mutable_settings()->CopyFrom(settings); + metadata.set_progress(import.GetProgress()); + metadata.mutable_items_progress()->CopyFrom(import.GetItemsProgress()); + operation.mutable_metadata()->PackFrom(metadata); + + TResult result; + operation.mutable_result()->PackFrom(result); + } + +}; // TImportConv + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_import_data.cpp b/ydb/core/grpc_services/rpc_import_data.cpp index 2d1f1f8c7ea..8df9b3e2c73 100644 --- a/ydb/core/grpc_services/rpc_import_data.cpp +++ b/ydb/core/grpc_services/rpc_import_data.cpp @@ -1,8 +1,8 @@ -#include "grpc_request_proxy.h" -#include "rpc_calls.h" -#include "rpc_common.h" -#include "rpc_request_base.h" - +#include "grpc_request_proxy.h" +#include "rpc_calls.h" +#include "rpc_common.h" +#include "rpc_request_base.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/scheme/scheme_tabledefs.h> @@ -10,281 +10,281 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/io_formats/csv.h> - -#include <library/cpp/actors/core/hfunc.h> - + +#include <library/cpp/actors/core/hfunc.h> + #include <ydb/library/yql/public/udf/udf_types.h> - -#include <util/datetime/base.h> -#include <util/generic/hash.h> -#include <util/generic/maybe.h> -#include <util/generic/vector.h> -#include <util/memory/pool.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; -using namespace NKikimrIssues; -using namespace Ydb; - -class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataRequest, true> { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TResolve = NSchemeCache::TSchemeCacheRequest; - using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; - - static constexpr TDuration MAX_TIMEOUT = TDuration::Minutes(5); - - static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { - TVector<NUdf::TDataTypeId> result; - - for (const auto& [_, column] : entry.Columns) { - if (column.KeyOrder < 0) { - continue; - } - - if (result.size() <= static_cast<ui32>(column.KeyOrder)) { - result.resize(column.KeyOrder + 1); - } - - result[column.KeyOrder] = column.PType; - } - - return result; - } - - static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { - TVector<TCell> fromValues(keyColumnsCount); - TVector<TCell> toValues; - return TSerializedTableRange(fromValues, true, toValues, false); - } - - static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { - const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); - return MakeHolder<TKeyDesc>( - entry.TableId, - GetFullRange(keyColumnTypes.size()).ToTableRange(), - TKeyDesc::ERowOperation::Update, - keyColumnTypes, - TVector<TKeyDesc::TColumnOp>() - ); - } - - static ui64 GetShardId(const TTableRange& range, const TKeyDesc* keyDesc) { - Y_VERIFY(range.Point); - Y_VERIFY(!keyDesc->Partitions.empty()); - - TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( - keyDesc->Partitions.begin(), keyDesc->Partitions.end(), true, - [&](const TKeyDesc::TPartitionInfo& partition, bool) { - const int cmp = CompareBorders<true, false>( - partition.Range->EndKeyPrefix.GetCells(), range.From, - partition.Range->IsInclusive || partition.Range->IsPoint, - range.InclusiveFrom || range.Point, keyDesc->KeyColumnTypes - ); - - return (cmp < 0); - } - ); - - Y_VERIFY(it != keyDesc->Partitions.end()); - return it->ShardId; - } - - /// Get proxy services - - void GetProxyServices() { - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); - Become(&TThis::StateGetProxyServices); - } - - STATEFN(StateGetProxyServices) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, Handle); - } - } - - void Handle(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/memory/pool.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; +using namespace NKikimrIssues; +using namespace Ydb; + +class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataRequest, true> { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TResolve = NSchemeCache::TSchemeCacheRequest; + using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; + + static constexpr TDuration MAX_TIMEOUT = TDuration::Minutes(5); + + static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { + TVector<NUdf::TDataTypeId> result; + + for (const auto& [_, column] : entry.Columns) { + if (column.KeyOrder < 0) { + continue; + } + + if (result.size() <= static_cast<ui32>(column.KeyOrder)) { + result.resize(column.KeyOrder + 1); + } + + result[column.KeyOrder] = column.PType; + } + + return result; + } + + static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { + TVector<TCell> fromValues(keyColumnsCount); + TVector<TCell> toValues; + return TSerializedTableRange(fromValues, true, toValues, false); + } + + static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { + const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); + return MakeHolder<TKeyDesc>( + entry.TableId, + GetFullRange(keyColumnTypes.size()).ToTableRange(), + TKeyDesc::ERowOperation::Update, + keyColumnTypes, + TVector<TKeyDesc::TColumnOp>() + ); + } + + static ui64 GetShardId(const TTableRange& range, const TKeyDesc* keyDesc) { + Y_VERIFY(range.Point); + Y_VERIFY(!keyDesc->Partitions.empty()); + + TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( + keyDesc->Partitions.begin(), keyDesc->Partitions.end(), true, + [&](const TKeyDesc::TPartitionInfo& partition, bool) { + const int cmp = CompareBorders<true, false>( + partition.Range->EndKeyPrefix.GetCells(), range.From, + partition.Range->IsInclusive || partition.Range->IsPoint, + range.InclusiveFrom || range.Point, keyDesc->KeyColumnTypes + ); + + return (cmp < 0); + } + ); + + Y_VERIFY(it != keyDesc->Partitions.end()); + return it->ShardId; + } + + /// Get proxy services + + void GetProxyServices() { + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); + Become(&TThis::StateGetProxyServices); + } + + STATEFN(StateGetProxyServices) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, Handle); + } + } + + void Handle(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { LeaderPipeCache = ev->Get()->Services.LeaderPipeCache; - ResolvePath(); - } - - /// Resolve path - - void ResolvePath() { - auto request = MakeHolder<TNavigate>(); - request->DatabaseName = NKikimr::CanonizePath(DatabaseName); - - auto& entry = request->ResultSet.emplace_back(); - entry.Operation = TNavigate::OpTable; - entry.Path = NKikimr::SplitPath(Request->GetProtoRequest()->path()); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolvePath); - } - - STATEFN(StateResolvePath) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - } - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& request = ev->Get()->Request; - - if (request->ResultSet.empty()) { - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - const auto& entry = request->ResultSet.front(); - - switch (entry.Status) { - case TNavigate::EStatus::Ok: - break; - case TNavigate::EStatus::RootUnknown: - case TNavigate::EStatus::PathErrorUnknown: - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::PATH_NOT_EXIST); - case TNavigate::EStatus::LookupError: - case TNavigate::EStatus::RedirectLookupError: - return Reply(StatusIds::UNAVAILABLE, TIssuesIds::RESOLVE_LOOKUP_ERROR); - default: - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - if (!CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, NACLib::UpdateRow)) { - return; - } - - if (entry.Indexes) { - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DEFAULT_ERROR, - "Table should not have global secondary indexes"); - } - - for (const auto& [_, column] : entry.Columns) { - Y_VERIFY(Columns.emplace(column.Name, column).second); - } - - KeyDesc = MakeKeyDesc(entry); - ResolveKeys(); - } - - /// Resolve keys - - void ResolveKeys() { - auto request = MakeHolder<TResolve>(); - request->DatabaseName = NKikimr::CanonizePath(DatabaseName); - + ResolvePath(); + } + + /// Resolve path + + void ResolvePath() { + auto request = MakeHolder<TNavigate>(); + request->DatabaseName = NKikimr::CanonizePath(DatabaseName); + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = TNavigate::OpTable; + entry.Path = NKikimr::SplitPath(Request->GetProtoRequest()->path()); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolvePath); + } + + STATEFN(StateResolvePath) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + } + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + if (request->ResultSet.empty()) { + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + const auto& entry = request->ResultSet.front(); + + switch (entry.Status) { + case TNavigate::EStatus::Ok: + break; + case TNavigate::EStatus::RootUnknown: + case TNavigate::EStatus::PathErrorUnknown: + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::PATH_NOT_EXIST); + case TNavigate::EStatus::LookupError: + case TNavigate::EStatus::RedirectLookupError: + return Reply(StatusIds::UNAVAILABLE, TIssuesIds::RESOLVE_LOOKUP_ERROR); + default: + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + if (!CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, NACLib::UpdateRow)) { + return; + } + + if (entry.Indexes) { + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DEFAULT_ERROR, + "Table should not have global secondary indexes"); + } + + for (const auto& [_, column] : entry.Columns) { + Y_VERIFY(Columns.emplace(column.Name, column).second); + } + + KeyDesc = MakeKeyDesc(entry); + ResolveKeys(); + } + + /// Resolve keys + + void ResolveKeys() { + auto request = MakeHolder<TResolve>(); + request->DatabaseName = NKikimr::CanonizePath(DatabaseName); + request->ResultSet.emplace_back(std::move(KeyDesc)); - request->ResultSet.back().Access = NACLib::UpdateRow; - - Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); - Become(&TThis::StateResolveKeys); - } - - STATEFN(StateResolveKeys) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, Handle); - } - } - - void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - const auto& request = ev->Get()->Request; - - if (request->ResultSet.empty()) { - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - + request->ResultSet.back().Access = NACLib::UpdateRow; + + Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); + Become(&TThis::StateResolveKeys); + } + + STATEFN(StateResolveKeys) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, Handle); + } + } + + void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + if (request->ResultSet.empty()) { + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + auto& entry = request->ResultSet.front(); KeyDesc = std::move(entry.KeyDescription); - - switch (entry.Status) { - case TResolve::EStatus::OkData: - break; - case TResolve::EStatus::NotMaterialized: - case TResolve::EStatus::PathErrorNotExist: - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::PATH_NOT_EXIST); - case TResolve::EStatus::LookupError: - return Reply(StatusIds::UNAVAILABLE, TIssuesIds::RESOLVE_LOOKUP_ERROR); - default: - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - + + switch (entry.Status) { + case TResolve::EStatus::OkData: + break; + case TResolve::EStatus::NotMaterialized: + case TResolve::EStatus::PathErrorNotExist: + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::PATH_NOT_EXIST); + case TResolve::EStatus::LookupError: + return Reply(StatusIds::UNAVAILABLE, TIssuesIds::RESOLVE_LOOKUP_ERROR); + default: + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + if (KeyDesc->Partitions.empty()) { - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - ProcessData(); - } - - /// Process data - - void ProcessData() { - const auto& request = *Request->GetProtoRequest(); - - auto ev = MakeHolder<TEvDataShard::TEvUploadRowsRequest>(); - ev->Record.SetTableId(KeyDesc->TableId.PathId.LocalPathId); - - const auto timeout = request.operation_params().has_operation_timeout() - ? Min(GetDuration(request.operation_params().operation_timeout()), MAX_TIMEOUT) - : MAX_TIMEOUT; - ev->Record.SetCancelDeadlineMs((AppData()->TimeProvider->Now() + timeout).MilliSeconds()); - - TMaybe<ui64> shardId; - switch (request.format_case()) { - case Import::ImportDataRequest::kYdbDump: - if (!FillColumnIds(request.ydb_dump().columns(), ev->Record)) { - return; - } - shardId = ProcessData(request.ydb_dump(), request.data(), ev->Record); - break; - default: - Y_FAIL("unreachable"); - } - - if (!shardId) { - return; - } - + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + ProcessData(); + } + + /// Process data + + void ProcessData() { + const auto& request = *Request->GetProtoRequest(); + + auto ev = MakeHolder<TEvDataShard::TEvUploadRowsRequest>(); + ev->Record.SetTableId(KeyDesc->TableId.PathId.LocalPathId); + + const auto timeout = request.operation_params().has_operation_timeout() + ? Min(GetDuration(request.operation_params().operation_timeout()), MAX_TIMEOUT) + : MAX_TIMEOUT; + ev->Record.SetCancelDeadlineMs((AppData()->TimeProvider->Now() + timeout).MilliSeconds()); + + TMaybe<ui64> shardId; + switch (request.format_case()) { + case Import::ImportDataRequest::kYdbDump: + if (!FillColumnIds(request.ydb_dump().columns(), ev->Record)) { + return; + } + shardId = ProcessData(request.ydb_dump(), request.data(), ev->Record); + break; + default: + Y_FAIL("unreachable"); + } + + if (!shardId) { + return; + } + Send(LeaderPipeCache, new TEvPipeCache::TEvForward(ev.Release(), *shardId, true), IEventHandle::FlagTrackDelivery); - Become(&TThis::StateProcessData); - } - - template <typename TCont> - bool FillColumnIds(const TCont& columnNames, NKikimrTxDataShard::TEvUploadRowsRequest& request) { - auto& rowScheme = *request.MutableRowScheme(); - - for (const auto& column : columnNames) { - const auto* info = Columns.FindPtr(column); - if (!info) { - Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, - TStringBuilder() << "Unknown column: " << column); - return false; - } - - if (info->KeyOrder != -1) { - auto& ids = *rowScheme.MutableKeyColumnIds(); - - if (ids.size() < (info->KeyOrder + 1)) { - ids.Resize(info->KeyOrder + 1, 0); - } - - ids[info->KeyOrder] = info->Id; - } else { - rowScheme.AddValueColumnIds(info->Id); - } - } - - return true; - } - - TMaybe<ui64> ProcessData(const Import::YdbDumpFormat& format, TStringBuf data, NKikimrTxDataShard::TEvUploadRowsRequest& request) { - TMaybe<ui64> shardId; - - TMaybe<TOwnedCellVec> prevKey; - TMemoryPool pool(256); - + Become(&TThis::StateProcessData); + } + + template <typename TCont> + bool FillColumnIds(const TCont& columnNames, NKikimrTxDataShard::TEvUploadRowsRequest& request) { + auto& rowScheme = *request.MutableRowScheme(); + + for (const auto& column : columnNames) { + const auto* info = Columns.FindPtr(column); + if (!info) { + Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, + TStringBuilder() << "Unknown column: " << column); + return false; + } + + if (info->KeyOrder != -1) { + auto& ids = *rowScheme.MutableKeyColumnIds(); + + if (ids.size() < (info->KeyOrder + 1)) { + ids.Resize(info->KeyOrder + 1, 0); + } + + ids[info->KeyOrder] = info->Id; + } else { + rowScheme.AddValueColumnIds(info->Id); + } + } + + return true; + } + + TMaybe<ui64> ProcessData(const Import::YdbDumpFormat& format, TStringBuf data, NKikimrTxDataShard::TEvUploadRowsRequest& request) { + TMaybe<ui64> shardId; + + TMaybe<TOwnedCellVec> prevKey; + TMemoryPool pool(256); + std::vector<std::pair<i32, ui32>> columnOrderTypes; // {keyOrder, PType} columnOrderTypes.reserve(format.columns().size()); for (const auto& column : format.columns()) { @@ -297,143 +297,143 @@ class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataReque columnOrderTypes.emplace_back(info->KeyOrder, info->PType); } - while (data) { - pool.Clear(); - - TStringBuf line = data.NextTok('\n'); - const TStringBuf origLine = line; - - if (!line) { - if (data) { - continue; - } - - break; - } - - TVector<TCell> keys; - TVector<TCell> values; + while (data) { + pool.Clear(); + + TStringBuf line = data.NextTok('\n'); + const TStringBuf origLine = line; + + if (!line) { + if (data) { + continue; + } + + break; + } + + TVector<TCell> keys; + TVector<TCell> values; TString strError; ui64 numBytes = 0; - + if (!NFormats::TYdbDump::ParseLine(line, columnOrderTypes, pool, keys, values, strError, numBytes)) { TString message = TStringBuilder() << strError << " on line: " << origLine; Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, message); return Nothing(); - } - - Y_VERIFY(!keys.empty()); - - // sorting constraint - if (!CheckSorted(prevKey, keys)) { - return Nothing(); - } - - prevKey = TOwnedCellVec::Make(keys); - - // same partition constraint - if (!shardId) { - shardId = GetShardId(TTableRange(keys), KeyDesc.Get()); - } - - if (!CheckSameShard(*shardId, keys)) { - return Nothing(); - } - - // ok - auto& row = *request.AddRows(); - row.SetKeyColumns(TSerializedCellVec::Serialize(keys)); - row.SetValueColumns(TSerializedCellVec::Serialize(values)); - } - - return shardId; - } - - bool CheckSorted(const TMaybe<TOwnedCellVec>& prevKey, const TVector<TCell>& key) { - if (!prevKey) { - return true; - } - - const int cmp = CompareTypedCellVectors(prevKey->data(), key.data(), - KeyDesc->KeyColumnTypes.data(), prevKey->size(), key.size()); - if (cmp <= 0) { - return true; - } - - Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Keys must be sorted"); - return false; - } - - bool CheckSameShard(ui64 prevShardId, const TVector<TCell>& key) { - const ui64 shardId = GetShardId(TTableRange(key), KeyDesc.Get()); - if (prevShardId == shardId) { - return true; - } - - Reply(StatusIds::PRECONDITION_FAILED, TIssuesIds::DEFAULT_ERROR, "All keys must be from the same partition"); - return false; - } - - STATEFN(StateProcessData) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvDataShard::TEvUploadRowsResponse, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - hFunc(TEvents::TEvUndelivered, Handle); - } - } - - void Handle(TEvDataShard::TEvUploadRowsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - switch (record.GetStatus()) { - case NKikimrTxDataShard::TError::OK: - return Reply(MakeOperation(StatusIds::SUCCESS)); - case NKikimrTxDataShard::TError::SCHEME_ERROR: - return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); - case NKikimrTxDataShard::TError::BAD_ARGUMENT: - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); - case NKikimrTxDataShard::TError::EXECUTION_CANCELLED: - return Reply(StatusIds::TIMEOUT, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); - case NKikimrTxDataShard::TError::WRONG_SHARD_STATE: - return Reply(StatusIds::OVERLOADED, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); - default: - return Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); - }; - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - Reply(StatusIds::UNAVAILABLE, TIssuesIds::SHARD_NOT_AVAILABLE, - TStringBuilder() << "Failed to connect to shard: " << ev->Get()->TabletId); - } - - void Handle(TEvents::TEvUndelivered::TPtr&) { - Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::DEFAULT_ERROR, "Pipe cache is not available"); - } - -public: - using TRpcRequestActor<TImportDataRPC, TEvImportDataRequest, true>::TRpcRequestActor; - - void Bootstrap() { - switch (Request->GetProtoRequest()->format_case()) { - case Import::ImportDataRequest::kYdbDump: - break; - default: - return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unknown format"); - } - - GetProxyServices(); - } - -private: + } + + Y_VERIFY(!keys.empty()); + + // sorting constraint + if (!CheckSorted(prevKey, keys)) { + return Nothing(); + } + + prevKey = TOwnedCellVec::Make(keys); + + // same partition constraint + if (!shardId) { + shardId = GetShardId(TTableRange(keys), KeyDesc.Get()); + } + + if (!CheckSameShard(*shardId, keys)) { + return Nothing(); + } + + // ok + auto& row = *request.AddRows(); + row.SetKeyColumns(TSerializedCellVec::Serialize(keys)); + row.SetValueColumns(TSerializedCellVec::Serialize(values)); + } + + return shardId; + } + + bool CheckSorted(const TMaybe<TOwnedCellVec>& prevKey, const TVector<TCell>& key) { + if (!prevKey) { + return true; + } + + const int cmp = CompareTypedCellVectors(prevKey->data(), key.data(), + KeyDesc->KeyColumnTypes.data(), prevKey->size(), key.size()); + if (cmp <= 0) { + return true; + } + + Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Keys must be sorted"); + return false; + } + + bool CheckSameShard(ui64 prevShardId, const TVector<TCell>& key) { + const ui64 shardId = GetShardId(TTableRange(key), KeyDesc.Get()); + if (prevShardId == shardId) { + return true; + } + + Reply(StatusIds::PRECONDITION_FAILED, TIssuesIds::DEFAULT_ERROR, "All keys must be from the same partition"); + return false; + } + + STATEFN(StateProcessData) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvDataShard::TEvUploadRowsResponse, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(TEvents::TEvUndelivered, Handle); + } + } + + void Handle(TEvDataShard::TEvUploadRowsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + switch (record.GetStatus()) { + case NKikimrTxDataShard::TError::OK: + return Reply(MakeOperation(StatusIds::SUCCESS)); + case NKikimrTxDataShard::TError::SCHEME_ERROR: + return Reply(StatusIds::SCHEME_ERROR, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); + case NKikimrTxDataShard::TError::BAD_ARGUMENT: + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); + case NKikimrTxDataShard::TError::EXECUTION_CANCELLED: + return Reply(StatusIds::TIMEOUT, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); + case NKikimrTxDataShard::TError::WRONG_SHARD_STATE: + return Reply(StatusIds::OVERLOADED, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); + default: + return Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::DEFAULT_ERROR, record.GetErrorDescription()); + }; + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + Reply(StatusIds::UNAVAILABLE, TIssuesIds::SHARD_NOT_AVAILABLE, + TStringBuilder() << "Failed to connect to shard: " << ev->Get()->TabletId); + } + + void Handle(TEvents::TEvUndelivered::TPtr&) { + Reply(StatusIds::INTERNAL_ERROR, TIssuesIds::DEFAULT_ERROR, "Pipe cache is not available"); + } + +public: + using TRpcRequestActor<TImportDataRPC, TEvImportDataRequest, true>::TRpcRequestActor; + + void Bootstrap() { + switch (Request->GetProtoRequest()->format_case()) { + case Import::ImportDataRequest::kYdbDump: + break; + default: + return Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Unknown format"); + } + + GetProxyServices(); + } + +private: TActorId LeaderPipeCache; - THashMap<TString, TSysTables::TTableColumnInfo> Columns; - THolder<TKeyDesc> KeyDesc; - -}; // TImportDataRPC - -void TGRpcRequestProxy::Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TImportDataRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + THashMap<TString, TSysTables::TTableColumnInfo> Columns; + THolder<TKeyDesc> KeyDesc; + +}; // TImportDataRPC + +void TGRpcRequestProxy::Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TImportDataRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_kh_describe.cpp b/ydb/core/grpc_services/rpc_kh_describe.cpp index bf4aaae4551..e2c48213ced 100644 --- a/ydb/core/grpc_services/rpc_kh_describe.cpp +++ b/ydb/core/grpc_services/rpc_kh_describe.cpp @@ -159,7 +159,7 @@ private: const auto& entry = ResolveNamesResult->ResultSet.front(); if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, ToString(entry.Status), ctx); + return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, ToString(entry.Status), ctx); } TString errorMessage; diff --git a/ydb/core/grpc_services/rpc_kh_snapshots.cpp b/ydb/core/grpc_services/rpc_kh_snapshots.cpp index 003c8b3e284..c390c81faeb 100644 --- a/ydb/core/grpc_services/rpc_kh_snapshots.cpp +++ b/ydb/core/grpc_services/rpc_kh_snapshots.cpp @@ -2,7 +2,7 @@ #include "resolve_local_db_table.h" #include "rpc_calls.h" -#include "rpc_common.h" +#include "rpc_common.h" #include "rpc_deferrable.h" #include <ydb/core/actorlib_impl/long_timer.h> @@ -101,8 +101,8 @@ public: req->Record.SetUserToken(token); } - SetDatabase(req.Get(), Request()); - + SetDatabase(req.Get(), Request()); + auto* tx = req->Record.MutableTransaction()->MutableCreateVolatileSnapshot(); for (const TString& path : proto->path()) { if (proto->ignore_system_views() && TryParseLocalDbPath(::NKikimr::SplitPath(path))) { @@ -237,8 +237,8 @@ public: req->Record.SetUserToken(token); } - SetDatabase(req.Get(), Request()); - + SetDatabase(req.Get(), Request()); + auto* tx = req->Record.MutableTransaction()->MutableRefreshVolatileSnapshot(); for (const TString& path : proto->path()) { if (proto->ignore_system_views() && TryParseLocalDbPath(::NKikimr::SplitPath(path))) { @@ -378,8 +378,8 @@ public: req->Record.SetUserToken(token); } - SetDatabase(req.Get(), Request()); - + SetDatabase(req.Get(), Request()); + auto* tx = req->Record.MutableTransaction()->MutableDiscardVolatileSnapshot(); for (const TString& path : proto->path()) { if (proto->ignore_system_views() && TryParseLocalDbPath(::NKikimr::SplitPath(path))) { diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h index dfe9b160f9b..2811a49e78a 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.h +++ b/ydb/core/grpc_services/rpc_kqp_base.h @@ -19,7 +19,7 @@ inline TString DecodePreparedQueryId(const TString& in) { throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) << "got empty preparedQueryId message"; } - NOperationId::TOperationId opId(in); + NOperationId::TOperationId opId(in); const auto& ids = opId.GetValue("id"); if (ids.size() != 1) { throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) diff --git a/ydb/core/grpc_services/rpc_list_operations.cpp b/ydb/core/grpc_services/rpc_list_operations.cpp index 71f548d733b..a905efd8d6f 100644 --- a/ydb/core/grpc_services/rpc_list_operations.cpp +++ b/ydb/core/grpc_services/rpc_list_operations.cpp @@ -1,80 +1,80 @@ -#include "grpc_request_proxy.h" +#include "grpc_request_proxy.h" #include "operation_helpers.h" -#include "rpc_export_base.h" -#include "rpc_import_base.h" -#include "rpc_calls.h" -#include "rpc_operation_request_base.h" - +#include "rpc_export_base.h" +#include "rpc_import_base.h" +#include "rpc_calls.h" +#include "rpc_operation_request_base.h" + #include <ydb/core/tx/schemeshard/schemeshard_build_index.h> #include <ydb/core/tx/schemeshard/schemeshard_export.h> #include <ydb/core/tx/schemeshard/schemeshard_import.h> #include <ydb/public/lib/operation_id/operation_id.h> - + #include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; using namespace NSchemeShard; -using namespace NKikimrIssues; -using namespace NOperationId; -using namespace Ydb; - -class TListOperationsRPC: public TRpcOperationRequestActor<TListOperationsRPC, TEvListOperationsRequest>, - public TExportConv { - - TStringBuf GetLogPrefix() const override { - switch (ParseKind(Request->GetProtoRequest()->kind())) { - case TOperationId::EXPORT: - return "[ListExports]"; - case TOperationId::IMPORT: - return "[ListImports]"; - case TOperationId::BUILD_INDEX: - return "[ListIndexBuilds]"; - default: - return "[Untagged]"; +using namespace NKikimrIssues; +using namespace NOperationId; +using namespace Ydb; + +class TListOperationsRPC: public TRpcOperationRequestActor<TListOperationsRPC, TEvListOperationsRequest>, + public TExportConv { + + TStringBuf GetLogPrefix() const override { + switch (ParseKind(Request->GetProtoRequest()->kind())) { + case TOperationId::EXPORT: + return "[ListExports]"; + case TOperationId::IMPORT: + return "[ListImports]"; + case TOperationId::BUILD_INDEX: + return "[ListIndexBuilds]"; + default: + return "[Untagged]"; } } - IEventBase* MakeRequest() override { - const auto& request = *Request->GetProtoRequest(); - - switch (ParseKind(Request->GetProtoRequest()->kind())) { - case TOperationId::EXPORT: - return new TEvExport::TEvListExportsRequest(DatabaseName, request.page_size(), request.page_token(), request.kind()); - case TOperationId::IMPORT: - return new TEvImport::TEvListImportsRequest(DatabaseName, request.page_size(), request.page_token(), request.kind()); - case TOperationId::BUILD_INDEX: - return new TEvIndexBuilder::TEvListRequest(DatabaseName, request.page_size(), request.page_token()); - default: - Y_FAIL("unreachable"); - } - } - - void Handle(TEvExport::TEvListExportsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); - - LOG_D("Handle TEvExport::TEvListExportsResponse" - << ": record# " << record.ShortDebugString()); - - TResponse response; - response.set_status(record.GetStatus()); - if (record.GetIssues().size()) { - response.mutable_issues()->CopyFrom(record.GetIssues()); + IEventBase* MakeRequest() override { + const auto& request = *Request->GetProtoRequest(); + + switch (ParseKind(Request->GetProtoRequest()->kind())) { + case TOperationId::EXPORT: + return new TEvExport::TEvListExportsRequest(DatabaseName, request.page_size(), request.page_token(), request.kind()); + case TOperationId::IMPORT: + return new TEvImport::TEvListImportsRequest(DatabaseName, request.page_size(), request.page_token(), request.kind()); + case TOperationId::BUILD_INDEX: + return new TEvIndexBuilder::TEvListRequest(DatabaseName, request.page_size(), request.page_token()); + default: + Y_FAIL("unreachable"); } - for (const auto& entry : record.GetEntries()) { - *response.add_operations() = TExportConv::ToOperation(entry); + } + + void Handle(TEvExport::TEvListExportsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); + + LOG_D("Handle TEvExport::TEvListExportsResponse" + << ": record# " << record.ShortDebugString()); + + TResponse response; + response.set_status(record.GetStatus()); + if (record.GetIssues().size()) { + response.mutable_issues()->CopyFrom(record.GetIssues()); } - response.set_next_page_token(record.GetNextPageToken()); - Reply(response); - } + for (const auto& entry : record.GetEntries()) { + *response.add_operations() = TExportConv::ToOperation(entry); + } + response.set_next_page_token(record.GetNextPageToken()); + Reply(response); + } - void Handle(TEvImport::TEvListImportsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record.GetResponse(); + void Handle(TEvImport::TEvListImportsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record.GetResponse(); - LOG_D("Handle TEvImport::TEvListImportsResponse" - << ": record# " << record.ShortDebugString()); + LOG_D("Handle TEvImport::TEvListImportsResponse" + << ": record# " << record.ShortDebugString()); TResponse response; response.set_status(record.GetStatus()); @@ -82,65 +82,65 @@ class TListOperationsRPC: public TRpcOperationRequestActor<TListOperationsRPC, T response.mutable_issues()->CopyFrom(record.GetIssues()); } for (const auto& entry : record.GetEntries()) { - *response.add_operations() = TImportConv::ToOperation(entry); + *response.add_operations() = TImportConv::ToOperation(entry); } response.set_next_page_token(record.GetNextPageToken()); Reply(response); } - void Handle(TEvIndexBuilder::TEvListResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - LOG_D("Handle TEvIndexBuilder::TEvListResponse" - << ": record# " << record.ShortDebugString()); - - TResponse response; - - response.set_status(record.GetStatus()); - if (record.GetIssues().size()) { - response.mutable_issues()->CopyFrom(record.GetIssues()); - } - for (const auto& entry : record.GetEntries()) { - auto operation = response.add_operations(); - ::NKikimr::NGRpcService::ToOperation(entry, operation); - } - response.set_next_page_token(record.GetNextPageToken()); - Reply(response); - } - -public: - using TRpcOperationRequestActor::TRpcOperationRequestActor; - - void Bootstrap() { - switch (ParseKind(Request->GetProtoRequest()->kind())) { - case TOperationId::EXPORT: - case TOperationId::IMPORT: - case TOperationId::BUILD_INDEX: - break; - - default: - return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); - } - - ResolveDatabase(); - Become(&TListOperationsRPC::StateWait); - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExport::TEvListExportsResponse, Handle); - hFunc(TEvImport::TEvListImportsResponse, Handle); - hFunc(TEvIndexBuilder::TEvListResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -}; // TListOperationsRPC - -void TGRpcRequestProxy::Handle(TEvListOperationsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TListOperationsRPC(ev->Release().Release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + void Handle(TEvIndexBuilder::TEvListResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + LOG_D("Handle TEvIndexBuilder::TEvListResponse" + << ": record# " << record.ShortDebugString()); + + TResponse response; + + response.set_status(record.GetStatus()); + if (record.GetIssues().size()) { + response.mutable_issues()->CopyFrom(record.GetIssues()); + } + for (const auto& entry : record.GetEntries()) { + auto operation = response.add_operations(); + ::NKikimr::NGRpcService::ToOperation(entry, operation); + } + response.set_next_page_token(record.GetNextPageToken()); + Reply(response); + } + +public: + using TRpcOperationRequestActor::TRpcOperationRequestActor; + + void Bootstrap() { + switch (ParseKind(Request->GetProtoRequest()->kind())) { + case TOperationId::EXPORT: + case TOperationId::IMPORT: + case TOperationId::BUILD_INDEX: + break; + + default: + return Reply(StatusIds::UNSUPPORTED, TIssuesIds::DEFAULT_ERROR, "Unknown operation kind"); + } + + ResolveDatabase(); + Become(&TListOperationsRPC::StateWait); + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExport::TEvListExportsResponse, Handle); + hFunc(TEvImport::TEvListImportsResponse, Handle); + hFunc(TEvIndexBuilder::TEvListResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + +}; // TListOperationsRPC + +void TGRpcRequestProxy::Handle(TEvListOperationsRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TListOperationsRPC(ev->Release().Release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_make_directory.cpp b/ydb/core/grpc_services/rpc_make_directory.cpp index 37e30df2764..f6e6b2d0cd5 100644 --- a/ydb/core/grpc_services/rpc_make_directory.cpp +++ b/ydb/core/grpc_services/rpc_make_directory.cpp @@ -34,7 +34,7 @@ private: const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; try { - pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); + pathPair = SplitPath(Request_->GetDatabaseName(), req->path()); } catch (const std::exception& ex) { Request_->RaiseIssue(NYql::ExceptionToIssue(ex)); return ReplyWithResult(StatusIds::BAD_REQUEST, ctx); @@ -43,7 +43,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_modify_permissions.cpp b/ydb/core/grpc_services/rpc_modify_permissions.cpp index 13ee2bbfcba..d9b01887d2c 100644 --- a/ydb/core/grpc_services/rpc_modify_permissions.cpp +++ b/ydb/core/grpc_services/rpc_modify_permissions.cpp @@ -46,7 +46,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpModifyACL); diff --git a/ydb/core/grpc_services/rpc_operation_request_base.h b/ydb/core/grpc_services/rpc_operation_request_base.h index 3c97cf6e244..697d2109cd7 100644 --- a/ydb/core/grpc_services/rpc_operation_request_base.h +++ b/ydb/core/grpc_services/rpc_operation_request_base.h @@ -1,147 +1,147 @@ -#pragma once - -#include "rpc_request_base.h" - +#pragma once + +#include "rpc_request_base.h" + #include <ydb/core/base/path.h> #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> - -namespace NKikimr { -namespace NGRpcService { - -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) - -template <typename TDerived, typename TEvRequest, bool HasOperation = false> -class TRpcOperationRequestActor: public TRpcRequestActor<TDerived, TEvRequest, HasOperation> { -protected: - virtual TStringBuf GetLogPrefix() const = 0; - virtual IEventBase* MakeRequest() = 0; - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - } - } - - void AllocateTxId() { - LOG_D("Allocate txId"); - this->Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); - } - - void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { - TxId = ev->Get()->TxId; - - LOG_D("TEvTxUserProxy::TEvAllocateTxIdResult"); - ResolveDatabase(); - } - - void ResolveDatabase() { - LOG_D("Resolve database" - << ": name# " << this->DatabaseName); - - auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - request->DatabaseName = this->DatabaseName; - - auto& entry = request->ResultSet.emplace_back(); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - entry.Path = NKikimr::SplitPath(this->DatabaseName); - - this->Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& request = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (request->ResultSet.empty()) { - return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - const auto& entry = request->ResultSet.front(); - - if (request->ErrorCount > 0) { - switch (entry.Status) { - case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: - return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::PATH_NOT_EXIST); - case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: - case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: - return this->Reply(Ydb::StatusIds::UNAVAILABLE, NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR); - default: - return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); - } - } - - if (!this->CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, NACLib::GenericRead | NACLib::GenericWrite)) { - return; - } - - auto domainInfo = entry.DomainInfo; - if (!domainInfo) { - LOG_E("Got empty domain info"); - return this->Reply(Ydb::StatusIds::INTERNAL_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); - } - - SendRequest(domainInfo->ExtractSchemeShard()); - } - - void SendRequest(ui64 schemeShardId) { - LOG_D("Send request" - << ": schemeShardId# " << schemeShardId); - - if (!PipeClient) { - NTabletPipe::TClientConfig config; + +namespace NKikimr { +namespace NGRpcService { + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, GetLogPrefix() << " " << this->SelfId() << " [" << this->TxId << "] " << stream) + +template <typename TDerived, typename TEvRequest, bool HasOperation = false> +class TRpcOperationRequestActor: public TRpcRequestActor<TDerived, TEvRequest, HasOperation> { +protected: + virtual TStringBuf GetLogPrefix() const = 0; + virtual IEventBase* MakeRequest() = 0; + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + } + } + + void AllocateTxId() { + LOG_D("Allocate txId"); + this->Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); + } + + void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { + TxId = ev->Get()->TxId; + + LOG_D("TEvTxUserProxy::TEvAllocateTxIdResult"); + ResolveDatabase(); + } + + void ResolveDatabase() { + LOG_D("Resolve database" + << ": name# " << this->DatabaseName); + + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + request->DatabaseName = this->DatabaseName; + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = NKikimr::SplitPath(this->DatabaseName); + + this->Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (request->ResultSet.empty()) { + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + const auto& entry = request->ResultSet.front(); + + if (request->ErrorCount > 0) { + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::PATH_NOT_EXIST); + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + return this->Reply(Ydb::StatusIds::UNAVAILABLE, NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR); + default: + return this->Reply(Ydb::StatusIds::SCHEME_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + } + + if (!this->CheckAccess(CanonizePath(entry.Path), entry.SecurityObject, NACLib::GenericRead | NACLib::GenericWrite)) { + return; + } + + auto domainInfo = entry.DomainInfo; + if (!domainInfo) { + LOG_E("Got empty domain info"); + return this->Reply(Ydb::StatusIds::INTERNAL_ERROR, NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR); + } + + SendRequest(domainInfo->ExtractSchemeShard()); + } + + void SendRequest(ui64 schemeShardId) { + LOG_D("Send request" + << ": schemeShardId# " << schemeShardId); + + if (!PipeClient) { + NTabletPipe::TClientConfig config; config.RetryPolicy = {.RetryLimitCount = 3}; PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), schemeShardId, config)); - } - - NTabletPipe::SendData(this->SelfId(), PipeClient, MakeRequest()); - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - if (ev->Get()->Status != NKikimrProto::OK) { - DeliveryProblem(); - } - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&) { - DeliveryProblem(); - } - - void DeliveryProblem() { - LOG_W("Delivery problem"); - this->Reply(Ydb::StatusIds::UNAVAILABLE); - } - - void PassAway() override { - NTabletPipe::CloseAndForgetClient(this->SelfId(), PipeClient); - IActor::PassAway(); - } - -public: - using TRpcRequestActor<TDerived, TEvRequest, HasOperation>::TRpcRequestActor; - -protected: - ui64 TxId = 0; - -private: - TActorId PipeClient; - -}; // TRpcOperationRequestActor - -} // namespace NGRpcService -} // namespace NKikimr + } + + NTabletPipe::SendData(this->SelfId(), PipeClient, MakeRequest()); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->Status != NKikimrProto::OK) { + DeliveryProblem(); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&) { + DeliveryProblem(); + } + + void DeliveryProblem() { + LOG_W("Delivery problem"); + this->Reply(Ydb::StatusIds::UNAVAILABLE); + } + + void PassAway() override { + NTabletPipe::CloseAndForgetClient(this->SelfId(), PipeClient); + IActor::PassAway(); + } + +public: + using TRpcRequestActor<TDerived, TEvRequest, HasOperation>::TRpcRequestActor; + +protected: + ui64 TxId = 0; + +private: + TActorId PipeClient; + +}; // TRpcOperationRequestActor + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_prepare_data_query.cpp b/ydb/core/grpc_services/rpc_prepare_data_query.cpp index cd7dc4f4e54..eb4c5718774 100644 --- a/ydb/core/grpc_services/rpc_prepare_data_query.cpp +++ b/ydb/core/grpc_services/rpc_prepare_data_query.cpp @@ -15,7 +15,7 @@ namespace NKikimr { namespace NGRpcService { using namespace NActors; -using namespace NOperationId; +using namespace NOperationId; using namespace Ydb; using namespace NKqp; diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index dff855df3f1..4baae606a97 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -198,7 +198,7 @@ private: Y_VERIFY(ResolveNamesResult->ResultSet.size() == 1); const auto& entry = ResolveNamesResult->ResultSet.front(); if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, ToString(entry.Status), ctx); + return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, ToString(entry.Status), ctx); } TString errorMessage; diff --git a/ydb/core/grpc_services/rpc_remove_directory.cpp b/ydb/core/grpc_services/rpc_remove_directory.cpp index d0b0d9dae88..9bdc2ad7b20 100644 --- a/ydb/core/grpc_services/rpc_remove_directory.cpp +++ b/ydb/core/grpc_services/rpc_remove_directory.cpp @@ -42,7 +42,7 @@ private: const auto& workingDir = pathPair.first; const auto& name = pathPair.second; - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest = CreateProposeTransaction(); NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record; NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme(); modifyScheme->SetWorkingDir(workingDir); diff --git a/ydb/core/grpc_services/rpc_request_base.h b/ydb/core/grpc_services/rpc_request_base.h index 692b5f4522d..b5cb84fc0b0 100644 --- a/ydb/core/grpc_services/rpc_request_base.h +++ b/ydb/core/grpc_services/rpc_request_base.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/core/base/appdata.h> #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/protos/services.pb.h> @@ -8,160 +8,160 @@ #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> #include <ydb/public/api/protos/ydb_issue_message.pb.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> - -#include <util/generic/ptr.h> - -namespace NKikimr { -namespace NGRpcService { - -using namespace NActors; - -template <typename TDerived, typename TEvRequest, bool HasOperation = false> -class TRpcRequestActor: public TActorBootstrapped<TDerived> { -public: - using TRequest = typename TEvRequest::TRequest; - using TResponse = typename TEvRequest::TResponse; - using TOperation = Ydb::Operations::Operation; - -protected: - void Reply( - const TResponse& response, - const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS - ) { + +#include <util/generic/ptr.h> + +namespace NKikimr { +namespace NGRpcService { + +using namespace NActors; + +template <typename TDerived, typename TEvRequest, bool HasOperation = false> +class TRpcRequestActor: public TActorBootstrapped<TDerived> { +public: + using TRequest = typename TEvRequest::TRequest; + using TResponse = typename TEvRequest::TResponse; + using TOperation = Ydb::Operations::Operation; + +protected: + void Reply( + const TResponse& response, + const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS + ) { TProtoResponseHelper::SendProtoResponse(response, status, Request); - this->PassAway(); - } - - void Reply( - const Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues - ) { - TResponse response; - if constexpr (HasOperation) { - *response.mutable_operation() = MakeOperation(status, issues); - } else { - response.set_status(status); - if (issues.size()) { - response.mutable_issues()->CopyFrom(issues); - } - } - Reply(response, status); - } - - void Reply(const Ydb::StatusIds::StatusCode status, const TString& error = TString()) { - Reply(status, ErrorToIssues(error)); - } - - void Reply( - const Ydb::StatusIds::StatusCode status, - NKikimrIssues::TIssuesIds::EIssueCode code, - const TString& error - ) { - Request->RaiseIssue(MakeIssue(code, error)); + this->PassAway(); + } + + void Reply( + const Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues + ) { + TResponse response; + if constexpr (HasOperation) { + *response.mutable_operation() = MakeOperation(status, issues); + } else { + response.set_status(status); + if (issues.size()) { + response.mutable_issues()->CopyFrom(issues); + } + } + Reply(response, status); + } + + void Reply(const Ydb::StatusIds::StatusCode status, const TString& error = TString()) { + Reply(status, ErrorToIssues(error)); + } + + void Reply( + const Ydb::StatusIds::StatusCode status, + NKikimrIssues::TIssuesIds::EIssueCode code, + const TString& error + ) { + Request->RaiseIssue(MakeIssue(code, error)); Reply(status, error); - } - - void Reply(const Ydb::StatusIds::StatusCode status, NKikimrIssues::TIssuesIds::EIssueCode code) { - Request->RaiseIssue(MakeIssue(code)); - Reply(status); - } - - void Reply(const Ydb::Operations::Operation& operation) { + } + + void Reply(const Ydb::StatusIds::StatusCode status, NKikimrIssues::TIssuesIds::EIssueCode code) { + Request->RaiseIssue(MakeIssue(code)); + Reply(status); + } + + void Reply(const Ydb::Operations::Operation& operation) { Request->SendOperation(operation); this->PassAway(); - } - - bool CheckAccess(const TString& path, TIntrusivePtr<TSecurityObject> securityObject, ui32 access) { - if (!UserToken || !securityObject) { - return true; - } - - if (securityObject->CheckAccess(access, *UserToken)) { - return true; - } - - Reply(Ydb::StatusIds::UNAUTHORIZED, NKikimrIssues::TIssuesIds::ACCESS_DENIED, - TStringBuilder() << "Access denied" - << ": for# " << UserToken->GetUserSID() - << ", path# " << path - << ", access# " << NACLib::AccessRightsToString(access)); - return false; - } - - template <typename TResult, typename TMetadata> - static TOperation MakeOperation( - const TString& id, - const Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues, - const TResult& result, const TMetadata& metadata - ) { - TOperation operation = MakeOperationBase(id, true, status, issues); - operation.mutable_result()->PackFrom(result); - operation.mutable_metadata()->PackFrom(metadata); - return operation; - } - - static TOperation MakeOperation( - const Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues - ) { - return MakeOperationBase(TString(), true, status, issues); - } - - static TOperation MakeOperation(const Ydb::StatusIds::StatusCode status, const TString& error = TString()) { - return MakeOperation(status, ErrorToIssues(error)); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::GRPC_REQ; - } - - explicit TRpcRequestActor(TEvRequest* ev) - : Request(ev) - , DatabaseName(Request->GetDatabaseName().GetOrElse(DatabaseFromDomain(AppData()))) - { - if (const auto& userToken = Request->GetInternalToken()) { - UserToken = MakeHolder<NACLib::TUserToken>(userToken); - } - } - -private: - static google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> ErrorToIssues(const TString& error) { - google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues; - - if (error) { - auto& issue = *issues.Add(); - issue.set_severity(NYql::TSeverityIds::S_ERROR); - issue.set_message(error); - } - - return issues; - } - - static TOperation MakeOperationBase( - const TString& id, const bool ready, - const Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues - ) { - TOperation operation; - operation.set_id(id); - operation.set_ready(ready); - operation.set_status(status); - if (issues.size()) { - operation.mutable_issues()->CopyFrom(issues); - } - return operation; - } - -protected: - THolder<TEvRequest> Request; - const TString DatabaseName; - THolder<const NACLib::TUserToken> UserToken; - -}; // TRpcRequestActor - -} // namespace NGRpcService -} // namespace NKikimr + } + + bool CheckAccess(const TString& path, TIntrusivePtr<TSecurityObject> securityObject, ui32 access) { + if (!UserToken || !securityObject) { + return true; + } + + if (securityObject->CheckAccess(access, *UserToken)) { + return true; + } + + Reply(Ydb::StatusIds::UNAUTHORIZED, NKikimrIssues::TIssuesIds::ACCESS_DENIED, + TStringBuilder() << "Access denied" + << ": for# " << UserToken->GetUserSID() + << ", path# " << path + << ", access# " << NACLib::AccessRightsToString(access)); + return false; + } + + template <typename TResult, typename TMetadata> + static TOperation MakeOperation( + const TString& id, + const Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues, + const TResult& result, const TMetadata& metadata + ) { + TOperation operation = MakeOperationBase(id, true, status, issues); + operation.mutable_result()->PackFrom(result); + operation.mutable_metadata()->PackFrom(metadata); + return operation; + } + + static TOperation MakeOperation( + const Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues + ) { + return MakeOperationBase(TString(), true, status, issues); + } + + static TOperation MakeOperation(const Ydb::StatusIds::StatusCode status, const TString& error = TString()) { + return MakeOperation(status, ErrorToIssues(error)); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::GRPC_REQ; + } + + explicit TRpcRequestActor(TEvRequest* ev) + : Request(ev) + , DatabaseName(Request->GetDatabaseName().GetOrElse(DatabaseFromDomain(AppData()))) + { + if (const auto& userToken = Request->GetInternalToken()) { + UserToken = MakeHolder<NACLib::TUserToken>(userToken); + } + } + +private: + static google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> ErrorToIssues(const TString& error) { + google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues; + + if (error) { + auto& issue = *issues.Add(); + issue.set_severity(NYql::TSeverityIds::S_ERROR); + issue.set_message(error); + } + + return issues; + } + + static TOperation MakeOperationBase( + const TString& id, const bool ready, + const Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues + ) { + TOperation operation; + operation.set_id(id); + operation.set_ready(ready); + operation.set_status(status); + if (issues.size()) { + operation.mutable_issues()->CopyFrom(issues); + } + return operation; + } + +protected: + THolder<TEvRequest> Request; + const TString DatabaseName; + THolder<const NACLib::TUserToken> UserToken; + +}; // TRpcRequestActor + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_scheme_base.h b/ydb/core/grpc_services/rpc_scheme_base.h index 10b38ce1d57..0be68e802e1 100644 --- a/ydb/core/grpc_services/rpc_scheme_base.h +++ b/ydb/core/grpc_services/rpc_scheme_base.h @@ -29,14 +29,14 @@ protected: } } - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> CreateProposeTransaction() { - std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest(new TEvTxUserProxy::TEvProposeTransaction()); - SetAuthToken(proposeRequest, *this->Request_); - SetDatabase(proposeRequest.get(), *this->Request_); - SetRequestType(proposeRequest.get(), *this->Request_); - return proposeRequest; - } - + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> CreateProposeTransaction() { + std::unique_ptr<TEvTxUserProxy::TEvProposeTransaction> proposeRequest(new TEvTxUserProxy::TEvProposeTransaction()); + SetAuthToken(proposeRequest, *this->Request_); + SetDatabase(proposeRequest.get(), *this->Request_); + SetRequestType(proposeRequest.get(), *this->Request_); + return proposeRequest; + } + void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { if (ev->Get()->Status != NKikimrProto::OK) { NYql::TIssues issues; @@ -79,9 +79,9 @@ protected: NTabletPipe::SendData(ctx, SchemePipeActorId_, request.Release()); return; } - case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest: { - return this->ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, issueMessage, ctx); - } + case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest: { + return this->ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, issueMessage, ctx); + } case TEvTxUserProxy::TResultStatus::AccessDenied: { return this->ReplyWithResult(Ydb::StatusIds::UNAUTHORIZED, issueMessage, ctx); } diff --git a/ydb/core/grpc_services/table_profiles.cpp b/ydb/core/grpc_services/table_profiles.cpp index f8cc46c5b4a..367de4cc89e 100644 --- a/ydb/core/grpc_services/table_profiles.cpp +++ b/ydb/core/grpc_services/table_profiles.cpp @@ -170,7 +170,7 @@ bool TTableProfiles::ApplyTableProfile(const Ydb::Table::TableProfile &profile, partitioningPolicy.SetUniformPartitionsCount(policy.uniform_partitions()); break; case Ydb::Table::PartitioningPolicy::kExplicitPartitions: - if (!CopyExplicitPartitions(tableDesc, policy.explicit_partitions(), code, error)) + if (!CopyExplicitPartitions(tableDesc, policy.explicit_partitions(), code, error)) return false; break; default: diff --git a/ydb/core/grpc_services/table_settings.cpp b/ydb/core/grpc_services/table_settings.cpp index 5cf982477e1..97dc63a53c5 100644 --- a/ydb/core/grpc_services/table_settings.cpp +++ b/ydb/core/grpc_services/table_settings.cpp @@ -4,46 +4,46 @@ namespace NKikimr { namespace NGRpcService { bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, - Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings) { + const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, + Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings) { bool tableProfileSet = false; - if (in.has_profile()) { - const auto& profile = in.profile(); + if (in.has_profile()) { + const auto& profile = in.profile(); tableProfileSet = profile.preset_name() || profile.has_compaction_policy() || profile.has_execution_policy() || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() || profile.has_caching_policy(); } - - auto &partitionConfig = *out.MutablePartitionConfig(); - if (!in.compaction_policy().empty()) { + + auto &partitionConfig = *out.MutablePartitionConfig(); + if (!in.compaction_policy().empty()) { if (tableProfileSet) { MEWarning("CompactionPolicy", warnings); } - if (!profiles.ApplyCompactionPolicy(in.compaction_policy(), partitionConfig, code, error)) { + if (!profiles.ApplyCompactionPolicy(in.compaction_policy(), partitionConfig, code, error)) { return false; } } - return NKikimr::FillCreateTableSettingsDesc(out, in, code, error, warnings, tableProfileSet); -} + return NKikimr::FillCreateTableSettingsDesc(out, in, code, error, warnings, tableProfileSet); +} bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, - Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData) { + const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, + Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData) { bool changed = false; - auto &partitionConfig = *out.MutablePartitionConfig(); + auto &partitionConfig = *out.MutablePartitionConfig(); - if (in.set_compaction_policy()) { - if (!profiles.ApplyCompactionPolicy(in.set_compaction_policy(), partitionConfig, code, error, appData)) { + if (in.set_compaction_policy()) { + if (!profiles.ApplyCompactionPolicy(in.set_compaction_policy(), partitionConfig, code, error, appData)) { return false; } changed = true; } - return NKikimr::FillAlterTableSettingsDesc(out, in, code, error, changed); + return NKikimr::FillAlterTableSettingsDesc(out, in, code, error, changed); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/table_settings.h b/ydb/core/grpc_services/table_settings.h index 4f8b211cebe..c8c6d26281f 100644 --- a/ydb/core/grpc_services/table_settings.h +++ b/ydb/core/grpc_services/table_settings.h @@ -8,11 +8,11 @@ namespace NKikimr { namespace NGRpcService { bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, + const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings); bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, + const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData); } // namespace NGRpcService diff --git a/ydb/core/grpc_services/validation/helpers.cpp b/ydb/core/grpc_services/validation/helpers.cpp index 86bb1ff95aa..0b30fdcec79 100644 --- a/ydb/core/grpc_services/validation/helpers.cpp +++ b/ydb/core/grpc_services/validation/helpers.cpp @@ -1,66 +1,66 @@ -#include "helpers.h" - +#include "helpers.h" + #include <google/protobuf/compiler/cpp/cpp_helpers.h> #include <google/protobuf/any.pb.h> #include <google/protobuf/duration.pb.h> #include <google/protobuf/empty.pb.h> #include <google/protobuf/timestamp.pb.h> - -#include <util/string/builder.h> -#include <util/string/subst.h> - -namespace NKikimr { -namespace NValidation { - -static TString ProtoFileNameStripped(const google::protobuf::Descriptor* message) { - return google::protobuf::compiler::cpp::StripProto(message->file()->name()); -} - -TString HeaderFileName(const google::protobuf::Descriptor* message) { - return ProtoFileNameStripped(message).append(".pb.h"); -} - -TString SourceFileName(const google::protobuf::Descriptor* message) { - return ProtoFileNameStripped(message).append(".pb.cc"); -} - -TString ClassScope(const google::protobuf::Descriptor* message) { - return "class_scope:" + message->full_name(); -} - -TString NamespaceScope() { - return "namespace_scope"; -} - -TString ClassName(const google::protobuf::Descriptor* message) { - const TString ns = message->file()->package(); - TString className = !ns.empty() ? message->full_name().substr(ns.size() + 1) : message->full_name(); - SubstGlobal(className, ".", "::"); - return className; -} - -bool IsCustomMessage(const google::protobuf::Descriptor* message) { - if (!message) { - return false; - } - if (message->full_name() == google::protobuf::Any::descriptor()->full_name()) { - return false; - } - if (message->full_name() == google::protobuf::Duration::descriptor()->full_name()) { - return false; - } - if (message->full_name() == google::protobuf::Empty::descriptor()->full_name()) { - return false; - } - if (message->full_name() == google::protobuf::Timestamp::descriptor()->full_name()) { - return false; - } - if (message->options().map_entry()) { - return false; - } - - return true; -} - -} // NValidation -} // NKikimr + +#include <util/string/builder.h> +#include <util/string/subst.h> + +namespace NKikimr { +namespace NValidation { + +static TString ProtoFileNameStripped(const google::protobuf::Descriptor* message) { + return google::protobuf::compiler::cpp::StripProto(message->file()->name()); +} + +TString HeaderFileName(const google::protobuf::Descriptor* message) { + return ProtoFileNameStripped(message).append(".pb.h"); +} + +TString SourceFileName(const google::protobuf::Descriptor* message) { + return ProtoFileNameStripped(message).append(".pb.cc"); +} + +TString ClassScope(const google::protobuf::Descriptor* message) { + return "class_scope:" + message->full_name(); +} + +TString NamespaceScope() { + return "namespace_scope"; +} + +TString ClassName(const google::protobuf::Descriptor* message) { + const TString ns = message->file()->package(); + TString className = !ns.empty() ? message->full_name().substr(ns.size() + 1) : message->full_name(); + SubstGlobal(className, ".", "::"); + return className; +} + +bool IsCustomMessage(const google::protobuf::Descriptor* message) { + if (!message) { + return false; + } + if (message->full_name() == google::protobuf::Any::descriptor()->full_name()) { + return false; + } + if (message->full_name() == google::protobuf::Duration::descriptor()->full_name()) { + return false; + } + if (message->full_name() == google::protobuf::Empty::descriptor()->full_name()) { + return false; + } + if (message->full_name() == google::protobuf::Timestamp::descriptor()->full_name()) { + return false; + } + if (message->options().map_entry()) { + return false; + } + + return true; +} + +} // NValidation +} // NKikimr diff --git a/ydb/core/grpc_services/validation/helpers.h b/ydb/core/grpc_services/validation/helpers.h index 7a76990544f..60047a9dd78 100644 --- a/ydb/core/grpc_services/validation/helpers.h +++ b/ydb/core/grpc_services/validation/helpers.h @@ -1,19 +1,19 @@ -#pragma once - +#pragma once + #include <google/protobuf/descriptor.h> - -#include <util/generic/string.h> - -namespace NKikimr { -namespace NValidation { - -TString HeaderFileName(const google::protobuf::Descriptor* message); -TString SourceFileName(const google::protobuf::Descriptor* message); -TString ClassScope(const google::protobuf::Descriptor* message); -TString NamespaceScope(); -TString ClassName(const google::protobuf::Descriptor* message); - -bool IsCustomMessage(const google::protobuf::Descriptor* message); - -} // NValidation -} // NKikimr + +#include <util/generic/string.h> + +namespace NKikimr { +namespace NValidation { + +TString HeaderFileName(const google::protobuf::Descriptor* message); +TString SourceFileName(const google::protobuf::Descriptor* message); +TString ClassScope(const google::protobuf::Descriptor* message); +TString NamespaceScope(); +TString ClassName(const google::protobuf::Descriptor* message); + +bool IsCustomMessage(const google::protobuf::Descriptor* message); + +} // NValidation +} // NKikimr diff --git a/ydb/core/grpc_services/validation/main.cpp b/ydb/core/grpc_services/validation/main.cpp index 81b8092ca22..68c4bda65cb 100644 --- a/ydb/core/grpc_services/validation/main.cpp +++ b/ydb/core/grpc_services/validation/main.cpp @@ -1,581 +1,581 @@ -#include "helpers.h" - +#include "helpers.h" + #include <ydb/public/api//protos/annotations/validation.pb.h> #include <ydb/core/util/yverify_stream.h> - + #include <google/protobuf/compiler/code_generator.h> #include <google/protobuf/compiler/plugin.h> #include <google/protobuf/io/printer.h> #include <google/protobuf/io/zero_copy_stream.h> - -#include <util/generic/hash.h> -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/vector.h> -#include <util/string/builder.h> -#include <util/string/cast.h> -#include <util/string/subst.h> - -namespace NKikimr { -namespace NValidation { - -using namespace google::protobuf::compiler; -using namespace google::protobuf; - -using TVariables = std::map<TString, TString>; - -class TPrinter { -public: - explicit TPrinter(OutputDirectory* output, const TString& fileName, const TString& scope) - : Output(output) - , FileName(fileName) - , Scope(scope) - { - } - - io::Printer* operator->() { - if (!Printer) { - Stream.Reset(Output->OpenForInsert(FileName, Scope)); - Printer.ConstructInPlace(Stream.Get(), '$'); - } - - return Printer.Get(); - } - -private: - OutputDirectory* Output; - const TString FileName; - const TString Scope; - - THolder<io::ZeroCopyOutputStream> Stream; - TMaybe<io::Printer> Printer; - -}; // TPrinter - -class TFieldGenerator: public TThrRefBase { - void Required(TPrinter& printer) const { - Y_VERIFY(!Field->is_repeated(), "Repeated fields cannot be required or not"); - - if (Field->options().GetExtension(Ydb::required)) { - if (Field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) { - printer->Print(Vars, "if ($field$().empty()) {\n"); - } else if (Field->message_type()) { - printer->Print(Vars, "if (!has_$field$()) {\n"); - } else { - Y_FAIL_S(" Field type = " << Field->type_name() << " cannot be required or not"); - } - - printer->Indent(); - printer->Print(Vars, "__err = \"$field$ is required but not set\";\n"); - printer->Print(Vars, "return false;\n"); - printer->Outdent(); - printer->Print(Vars, "}\n"); - } - } - - void CheckLimit(TPrinter& printer, const Ydb::Limit& limit, TVariables vars) const { - switch (limit.kind_case()) { - case Ydb::Limit::kRange: - vars["min"] = ToString(limit.range().min()); - vars["max"] = ToString(limit.range().max()); - printer->Print(vars, "if (!($min$ <= $getter$ && $getter$ <= $max$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not in [$min$; $max$]\";\n"); - break; - - case Ydb::Limit::kLt: - vars["lt"] = ToString(limit.lt()); - printer->Print(vars, "if (!($getter$ < $lt$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not < $lt$\";\n"); - break; - - case Ydb::Limit::kLe: - vars["le"] = ToString(limit.le()); - printer->Print(vars, "if (!($getter$ <= $le$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not <= $le$\";\n"); - break; - - case Ydb::Limit::kEq: - vars["eq"] = ToString(limit.eq()); - printer->Print(vars, "if (!($getter$ == $eq$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not == $eq$\";\n"); - break; - - case Ydb::Limit::kGe: - vars["ge"] = ToString(limit.ge()); - printer->Print(vars, "if (!($getter$ >= $ge$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not >= $ge$\";\n"); - break; - - case Ydb::Limit::kGt: - vars["gt"] = ToString(limit.gt()); - printer->Print(vars, "if (!($getter$ > $gt$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not > $gt$\";\n"); - break; - - default: - Y_FAIL("Unknown limit type"); - } - - printer->Print(vars, "return false;\n"); - printer->Outdent(); - printer->Print(vars, "}\n"); - } - - static TString BuildValueChecker(const TString& getter, TStringBuf annValue) { - if (annValue.Contains('<') || annValue.Contains('>') || annValue.Contains('=')) { - return TStringBuilder() << getter << " " << annValue; - } - - if ((annValue.StartsWith('[') || annValue.StartsWith('(')) - && (annValue.EndsWith(']') || annValue.EndsWith(')')) - && annValue.Contains(';')) { - - const bool leftInclusive = annValue.StartsWith('['); - const bool rightInclusive = annValue.EndsWith(']'); - - annValue.Skip(1).Chop(1); - - const TStringBuf left = annValue.Before(';'); - const TStringBuf right = annValue.After(';'); - - return TStringBuilder() - << left << " " << (leftInclusive ? "<=" : "<") - << " " << getter << " && " << getter << " " - << (rightInclusive ? "<=" : "<") << " " << right; - } - - Y_FAIL_S("Invalid value: " << annValue); - } - - void CheckValue(TPrinter& printer, const FieldDescriptor* field, TVariables vars) const { - switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - case FieldDescriptor::CPPTYPE_INT64: - case FieldDescriptor::CPPTYPE_UINT32: - case FieldDescriptor::CPPTYPE_UINT64: - case FieldDescriptor::CPPTYPE_DOUBLE: - case FieldDescriptor::CPPTYPE_FLOAT: - case FieldDescriptor::CPPTYPE_BOOL: - case FieldDescriptor::CPPTYPE_STRING: - printer->Print(vars, "if (!($value_checker$)) {\n"); - printer->Indent(); - printer->Print(vars, "__err = \"$field$$kind$ is not $value$\";\n"); - printer->Print(vars, "return false;\n"); - printer->Outdent(); - printer->Print(vars, "}\n"); - break; - - default: - Y_FAIL_S("Cannot check value of field: type = " << field->type_name()); - } - } - - void Size(TPrinter& printer) const { - Y_VERIFY(Field->is_repeated(), "Cannot check size of non-repeated field"); - - TVariables vars = Vars; - vars["kind"] = " size"; - vars["getter"] = Field->name() + "_size()"; - - CheckLimit(printer, Field->options().GetExtension(Ydb::size), vars); - } - - void Length(TPrinter& printer) const { - if (Field->is_repeated()) { - printer->Print(Vars, "for (const auto& value : $field$()) {\n"); - printer->Indent(); - } - - if (Field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) { - TVariables vars = Vars; - vars["kind"] = "'s length"; - vars["getter"] = Field->is_repeated() ? "value.size()" : Field->name() + "().size()"; - - CheckLimit(printer, Field->options().GetExtension(Ydb::length), vars); - } else if (Field->is_map()) { - const FieldDescriptor* value = Field->message_type()->FindFieldByName("value"); - Y_VERIFY_S(value->cpp_type() == FieldDescriptor::CPPTYPE_STRING, "Cannot check length of field: type = " << value->type_name()); - - TVariables vars = Vars; - vars["kind"] = "'s value length"; - vars["getter"] = "value.second.size()"; - - CheckLimit(printer, Field->options().GetExtension(Ydb::length), vars); - } else { - Y_FAIL_S("Cannot check length of field: type = " << Field->type_name()); - } - - if (Field->is_repeated()) { - printer->Outdent(); - printer->Print(Vars, "}\n"); - } - } - - void MapKey(TPrinter& printer) const { - Y_VERIFY(Field->is_map(), "Cannot validate map key of non-map field"); - - printer->Print(Vars, "for (const auto& value : $field$()) {\n"); - printer->Indent(); - - TVariables vars = Vars; - - const auto& mapKey = Field->options().GetExtension(Ydb::map_key); - if (mapKey.has_length()) { - const FieldDescriptor* key = Field->message_type()->FindFieldByName("key"); - Y_VERIFY_S(key->cpp_type() == FieldDescriptor::CPPTYPE_STRING, "Cannot check length of field: type = " << key->type_name()); - - vars["kind"] = "'s key length"; - vars["getter"] = "value.first.size()"; - CheckLimit(printer, mapKey.length(), vars); - } - if (mapKey.value()) { - vars["kind"] = "'s key"; - vars["value"] = SubstGlobalCopy(mapKey.value(), '"', '\''); - vars["value_checker"] = BuildValueChecker("value.first", mapKey.value()); - CheckValue(printer, Field->message_type()->FindFieldByName("key"), vars); - } - - printer->Outdent(); - printer->Print(Vars, "}\n"); - } - - void Value(TPrinter& printer) const { - if (Field->is_repeated()) { - printer->Print(Vars, "for (const auto& value : $field$()) {\n"); - printer->Indent(); - } - - const TString value = Field->options().GetExtension(Ydb::value); - - TVariables vars = Vars; - vars["kind"] = "'s value"; - vars["value"] = SubstGlobalCopy(value, '"', '\''); - - if (Field->is_map()) { - vars["value_checker"] = BuildValueChecker("value.second", value); - CheckValue(printer, Field->message_type()->FindFieldByName("value"), vars); - } else { - vars["value_checker"] = BuildValueChecker(Field->is_repeated() ? "value" : Field->name() + "()", value); - CheckValue(printer, Field, vars); - } - - if (Field->is_repeated()) { - printer->Outdent(); - printer->Print(Vars, "}\n"); - } - } - - void Body(TPrinter& printer) const { - const auto& opts = Field->options(); - - if (opts.HasExtension(Ydb::required)) { - Required(printer); - } - if (opts.HasExtension(Ydb::size)) { - Size(printer); - } - if (opts.HasExtension(Ydb::length)) { - Length(printer); - } - if (opts.HasExtension(Ydb::map_key)) { - MapKey(printer); - } - if (opts.HasExtension(Ydb::value)) { - Value(printer); - } - - if (IsCustomMessage(Field->message_type())) { - if (Field->is_repeated()) { - printer->Print(Vars, "for (const auto& value : $field$()) {\n"); - printer->Indent(); - printer->Print(Vars, "if (!value.validate(__err)) {\n"); - printer->Indent(); - printer->Print(Vars, "return false;\n"); - printer->Outdent(); - printer->Print(Vars, "}\n"); - printer->Outdent(); - printer->Print(Vars, "}\n"); - printer->Print(Vars, "\nreturn true;\n"); - } else { - printer->Print(Vars, "return $field$().validate(__err);\n"); - } - } else { - printer->Print(Vars, "return true;\n"); - } - } - - bool HasValidators() const { - const auto& opts = Field->options(); - return opts.HasExtension(Ydb::required) - || opts.HasExtension(Ydb::size) - || opts.HasExtension(Ydb::length) - || opts.HasExtension(Ydb::map_key) - || opts.HasExtension(Ydb::value); - } - - static TString PascalName(TString camelName) { - camelName.to_upper(0, 1); - return camelName; - } - -public: - using TPtr = TIntrusivePtr<TFieldGenerator>; - - explicit TFieldGenerator(const FieldDescriptor* field, const TString& className) - : Field(field) - , Vars({ - {"class", className}, - {"func", "validate_" + field->name()}, - {"field", field->name()}, - {"PascalName", PascalName(field->camelcase_name())}, - }) - { - } - - const TVariables& GetVars() const { - return Vars; - } - - void Declare(TPrinter& printer) const { - if (!HasValidators() && !IsCustomMessage(Field->message_type())) { - printer->Print(Vars, "bool $func$(TProtoStringType&) const { return true; }\n"); - } else { - printer->Print(Vars, "bool $func$(TProtoStringType& __err) const;\n"); - } - } - - void Define(TPrinter& printer) const { - if (!HasValidators() && !IsCustomMessage(Field->message_type())) { - return; - } - - printer->Print(Vars, "bool $class$::$func$(TProtoStringType& __err) const {\n"); - printer->Indent(); - Body(printer); - printer->Outdent(); - printer->Print(Vars, "}\n\n"); - } - -private: - const FieldDescriptor* Field; - const TVariables Vars; - -}; // TFieldGenerator - -class TOneofGenerator: public TThrRefBase { - void Body(TPrinter& printer) const { - printer->Print(Vars, "switch ($oneof$_case()) {\n"); - - for (auto field : Fields) { - printer->Print(field->GetVars(), "case k$PascalName$:\n"); - printer->Indent(); - printer->Print(field->GetVars(), "return $func$(__err);\n"); - printer->Outdent(); - } - - printer->Print(Vars, "default:\n"); - printer->Indent(); - printer->Print(Vars, "return true;\n"); - printer->Outdent(); - - printer->Print(Vars, "}\n"); - } - -public: - using TPtr = TIntrusivePtr<TOneofGenerator>; - - explicit TOneofGenerator(const OneofDescriptor* oneof, const TString& className) - : Vars({ - {"class", className}, - {"oneof", oneof->name()}, - {"func", "validate_" + oneof->name()}, - }) - { - } - - const TVariables& GetVars() const { - return Vars; - } - - void AddField(TFieldGenerator::TPtr field) { - Fields.push_back(field); - } - - void Declare(TPrinter& printer) const { - printer->Print(Vars, "bool $func$(TProtoStringType& __err) const;\n"); - } - - void Define(TPrinter& printer) const { - printer->Print(Vars, "bool $class$::$func$(TProtoStringType& __err) const {\n"); - printer->Indent(); - Body(printer); - printer->Outdent(); - printer->Print(Vars, "}\n\n"); - } - -private: - const TVariables Vars; - TVector<TFieldGenerator::TPtr> Fields; - -}; // TOneofGenerator - -class TMessageGenerator { - struct TItem { - TOneofGenerator::TPtr Oneof; - TVector<TFieldGenerator::TPtr> Fields; - - explicit TItem(TOneofGenerator::TPtr oneof) - : Oneof(oneof) - { - } - - void AddField(TFieldGenerator::TPtr field) { - Fields.push_back(field); - if (Oneof) { - Oneof->AddField(field); - } - } - }; - - using TItems = THashMap<const OneofDescriptor*, TItem>; - - void Declare(TPrinter& printer, const TItems& items) { - if (!items) { - printer->Print(Vars, "bool validate(TProtoStringType&) const { return true; }\n"); - } else { - printer->Print(Vars, "bool validate(TProtoStringType& __err) const;\n"); - } - } - - void Define(TPrinter& printer, const TItems& items) { - if (!items) { - return; - } - - printer->Print(Vars, "bool $class$::validate(TProtoStringType& __err) const {\n"); - printer->Indent(); - printer->Print(Vars, "return\n"); - printer->Indent(); - - bool first = true; - for (const auto& [_, item] : items) { - if (!item.Oneof) { - for (const auto& field : item.Fields) { - if (first) { - first = false; - } else { - printer->Print(Vars, "&& "); - } - printer->Print(field->GetVars(), "$func$(__err)\n"); - } - } else { - if (first) { - first = false; - } else { - printer->Print(Vars, "&& "); - } - printer->Print(item.Oneof->GetVars(), "$func$(__err)\n"); - } - } - - printer->Outdent(); - printer->Print(Vars, ";\n"); - printer->Outdent(); - printer->Print(Vars, "}\n\n"); - } - -public: - explicit TMessageGenerator(const Descriptor* message, OutputDirectory* output) - : Message(message) - , Output(output) - , Header(output, HeaderFileName(message), ClassScope(message)) - , Source(output, SourceFileName(message), NamespaceScope()) - , Vars({ - {"class", ClassName(message)}, - }) - { - } - - void Generate() { - for (auto i = 0; i < Message->nested_type_count(); ++i) { - const Descriptor* message = Message->nested_type(i); - if (!IsCustomMessage(message)) { - continue; - } - - TMessageGenerator mg(message, Output); - mg.Generate(); - } - - TItems items; - - for (auto i = 0; i < Message->field_count(); ++i) { - const FieldDescriptor* field = Message->field(i); - const OneofDescriptor* oneof = field->containing_oneof(); - - TFieldGenerator::TPtr fieldGen = new TFieldGenerator(field, Vars.at("class")); - TOneofGenerator::TPtr oneofGen = oneof ? new TOneofGenerator(oneof, Vars.at("class")) : nullptr; - - auto it = items.find(oneof); - if (it == items.end()) { - it = items.emplace(oneof, TItem(oneofGen)).first; - } - - it->second.AddField(fieldGen); - } - - for (const auto& [_, item] : items) { - for (const auto& field : item.Fields) { - field->Declare(Header); - field->Define(Source); - } - - if (item.Oneof) { - item.Oneof->Declare(Header); - item.Oneof->Define(Source); - } - } - - Declare(Header, items); - Define(Source, items); - } - -private: - const Descriptor* Message; - OutputDirectory* Output; - TPrinter Header; - TPrinter Source; - const TVariables Vars; - -}; // TMessageGenerator - -class TCodeGenerator: public CodeGenerator { - bool Generate( - const FileDescriptor* file, - const TProtoStringType&, - OutputDirectory* output, - TProtoStringType*) const override final { - - for (auto i = 0; i < file->message_type_count(); ++i) { - TMessageGenerator mg(file->message_type(i), output); - mg.Generate(); - } - - return true; - } - -}; // TCodeGenerator - -} // NValidation -} // NKikimr - -int main(int argc, char* argv[]) { - NKikimr::NValidation::TCodeGenerator generator; - return google::protobuf::compiler::PluginMain(argc, argv, &generator); -} + +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/vector.h> +#include <util/string/builder.h> +#include <util/string/cast.h> +#include <util/string/subst.h> + +namespace NKikimr { +namespace NValidation { + +using namespace google::protobuf::compiler; +using namespace google::protobuf; + +using TVariables = std::map<TString, TString>; + +class TPrinter { +public: + explicit TPrinter(OutputDirectory* output, const TString& fileName, const TString& scope) + : Output(output) + , FileName(fileName) + , Scope(scope) + { + } + + io::Printer* operator->() { + if (!Printer) { + Stream.Reset(Output->OpenForInsert(FileName, Scope)); + Printer.ConstructInPlace(Stream.Get(), '$'); + } + + return Printer.Get(); + } + +private: + OutputDirectory* Output; + const TString FileName; + const TString Scope; + + THolder<io::ZeroCopyOutputStream> Stream; + TMaybe<io::Printer> Printer; + +}; // TPrinter + +class TFieldGenerator: public TThrRefBase { + void Required(TPrinter& printer) const { + Y_VERIFY(!Field->is_repeated(), "Repeated fields cannot be required or not"); + + if (Field->options().GetExtension(Ydb::required)) { + if (Field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) { + printer->Print(Vars, "if ($field$().empty()) {\n"); + } else if (Field->message_type()) { + printer->Print(Vars, "if (!has_$field$()) {\n"); + } else { + Y_FAIL_S(" Field type = " << Field->type_name() << " cannot be required or not"); + } + + printer->Indent(); + printer->Print(Vars, "__err = \"$field$ is required but not set\";\n"); + printer->Print(Vars, "return false;\n"); + printer->Outdent(); + printer->Print(Vars, "}\n"); + } + } + + void CheckLimit(TPrinter& printer, const Ydb::Limit& limit, TVariables vars) const { + switch (limit.kind_case()) { + case Ydb::Limit::kRange: + vars["min"] = ToString(limit.range().min()); + vars["max"] = ToString(limit.range().max()); + printer->Print(vars, "if (!($min$ <= $getter$ && $getter$ <= $max$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not in [$min$; $max$]\";\n"); + break; + + case Ydb::Limit::kLt: + vars["lt"] = ToString(limit.lt()); + printer->Print(vars, "if (!($getter$ < $lt$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not < $lt$\";\n"); + break; + + case Ydb::Limit::kLe: + vars["le"] = ToString(limit.le()); + printer->Print(vars, "if (!($getter$ <= $le$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not <= $le$\";\n"); + break; + + case Ydb::Limit::kEq: + vars["eq"] = ToString(limit.eq()); + printer->Print(vars, "if (!($getter$ == $eq$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not == $eq$\";\n"); + break; + + case Ydb::Limit::kGe: + vars["ge"] = ToString(limit.ge()); + printer->Print(vars, "if (!($getter$ >= $ge$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not >= $ge$\";\n"); + break; + + case Ydb::Limit::kGt: + vars["gt"] = ToString(limit.gt()); + printer->Print(vars, "if (!($getter$ > $gt$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not > $gt$\";\n"); + break; + + default: + Y_FAIL("Unknown limit type"); + } + + printer->Print(vars, "return false;\n"); + printer->Outdent(); + printer->Print(vars, "}\n"); + } + + static TString BuildValueChecker(const TString& getter, TStringBuf annValue) { + if (annValue.Contains('<') || annValue.Contains('>') || annValue.Contains('=')) { + return TStringBuilder() << getter << " " << annValue; + } + + if ((annValue.StartsWith('[') || annValue.StartsWith('(')) + && (annValue.EndsWith(']') || annValue.EndsWith(')')) + && annValue.Contains(';')) { + + const bool leftInclusive = annValue.StartsWith('['); + const bool rightInclusive = annValue.EndsWith(']'); + + annValue.Skip(1).Chop(1); + + const TStringBuf left = annValue.Before(';'); + const TStringBuf right = annValue.After(';'); + + return TStringBuilder() + << left << " " << (leftInclusive ? "<=" : "<") + << " " << getter << " && " << getter << " " + << (rightInclusive ? "<=" : "<") << " " << right; + } + + Y_FAIL_S("Invalid value: " << annValue); + } + + void CheckValue(TPrinter& printer, const FieldDescriptor* field, TVariables vars) const { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + case FieldDescriptor::CPPTYPE_INT64: + case FieldDescriptor::CPPTYPE_UINT32: + case FieldDescriptor::CPPTYPE_UINT64: + case FieldDescriptor::CPPTYPE_DOUBLE: + case FieldDescriptor::CPPTYPE_FLOAT: + case FieldDescriptor::CPPTYPE_BOOL: + case FieldDescriptor::CPPTYPE_STRING: + printer->Print(vars, "if (!($value_checker$)) {\n"); + printer->Indent(); + printer->Print(vars, "__err = \"$field$$kind$ is not $value$\";\n"); + printer->Print(vars, "return false;\n"); + printer->Outdent(); + printer->Print(vars, "}\n"); + break; + + default: + Y_FAIL_S("Cannot check value of field: type = " << field->type_name()); + } + } + + void Size(TPrinter& printer) const { + Y_VERIFY(Field->is_repeated(), "Cannot check size of non-repeated field"); + + TVariables vars = Vars; + vars["kind"] = " size"; + vars["getter"] = Field->name() + "_size()"; + + CheckLimit(printer, Field->options().GetExtension(Ydb::size), vars); + } + + void Length(TPrinter& printer) const { + if (Field->is_repeated()) { + printer->Print(Vars, "for (const auto& value : $field$()) {\n"); + printer->Indent(); + } + + if (Field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) { + TVariables vars = Vars; + vars["kind"] = "'s length"; + vars["getter"] = Field->is_repeated() ? "value.size()" : Field->name() + "().size()"; + + CheckLimit(printer, Field->options().GetExtension(Ydb::length), vars); + } else if (Field->is_map()) { + const FieldDescriptor* value = Field->message_type()->FindFieldByName("value"); + Y_VERIFY_S(value->cpp_type() == FieldDescriptor::CPPTYPE_STRING, "Cannot check length of field: type = " << value->type_name()); + + TVariables vars = Vars; + vars["kind"] = "'s value length"; + vars["getter"] = "value.second.size()"; + + CheckLimit(printer, Field->options().GetExtension(Ydb::length), vars); + } else { + Y_FAIL_S("Cannot check length of field: type = " << Field->type_name()); + } + + if (Field->is_repeated()) { + printer->Outdent(); + printer->Print(Vars, "}\n"); + } + } + + void MapKey(TPrinter& printer) const { + Y_VERIFY(Field->is_map(), "Cannot validate map key of non-map field"); + + printer->Print(Vars, "for (const auto& value : $field$()) {\n"); + printer->Indent(); + + TVariables vars = Vars; + + const auto& mapKey = Field->options().GetExtension(Ydb::map_key); + if (mapKey.has_length()) { + const FieldDescriptor* key = Field->message_type()->FindFieldByName("key"); + Y_VERIFY_S(key->cpp_type() == FieldDescriptor::CPPTYPE_STRING, "Cannot check length of field: type = " << key->type_name()); + + vars["kind"] = "'s key length"; + vars["getter"] = "value.first.size()"; + CheckLimit(printer, mapKey.length(), vars); + } + if (mapKey.value()) { + vars["kind"] = "'s key"; + vars["value"] = SubstGlobalCopy(mapKey.value(), '"', '\''); + vars["value_checker"] = BuildValueChecker("value.first", mapKey.value()); + CheckValue(printer, Field->message_type()->FindFieldByName("key"), vars); + } + + printer->Outdent(); + printer->Print(Vars, "}\n"); + } + + void Value(TPrinter& printer) const { + if (Field->is_repeated()) { + printer->Print(Vars, "for (const auto& value : $field$()) {\n"); + printer->Indent(); + } + + const TString value = Field->options().GetExtension(Ydb::value); + + TVariables vars = Vars; + vars["kind"] = "'s value"; + vars["value"] = SubstGlobalCopy(value, '"', '\''); + + if (Field->is_map()) { + vars["value_checker"] = BuildValueChecker("value.second", value); + CheckValue(printer, Field->message_type()->FindFieldByName("value"), vars); + } else { + vars["value_checker"] = BuildValueChecker(Field->is_repeated() ? "value" : Field->name() + "()", value); + CheckValue(printer, Field, vars); + } + + if (Field->is_repeated()) { + printer->Outdent(); + printer->Print(Vars, "}\n"); + } + } + + void Body(TPrinter& printer) const { + const auto& opts = Field->options(); + + if (opts.HasExtension(Ydb::required)) { + Required(printer); + } + if (opts.HasExtension(Ydb::size)) { + Size(printer); + } + if (opts.HasExtension(Ydb::length)) { + Length(printer); + } + if (opts.HasExtension(Ydb::map_key)) { + MapKey(printer); + } + if (opts.HasExtension(Ydb::value)) { + Value(printer); + } + + if (IsCustomMessage(Field->message_type())) { + if (Field->is_repeated()) { + printer->Print(Vars, "for (const auto& value : $field$()) {\n"); + printer->Indent(); + printer->Print(Vars, "if (!value.validate(__err)) {\n"); + printer->Indent(); + printer->Print(Vars, "return false;\n"); + printer->Outdent(); + printer->Print(Vars, "}\n"); + printer->Outdent(); + printer->Print(Vars, "}\n"); + printer->Print(Vars, "\nreturn true;\n"); + } else { + printer->Print(Vars, "return $field$().validate(__err);\n"); + } + } else { + printer->Print(Vars, "return true;\n"); + } + } + + bool HasValidators() const { + const auto& opts = Field->options(); + return opts.HasExtension(Ydb::required) + || opts.HasExtension(Ydb::size) + || opts.HasExtension(Ydb::length) + || opts.HasExtension(Ydb::map_key) + || opts.HasExtension(Ydb::value); + } + + static TString PascalName(TString camelName) { + camelName.to_upper(0, 1); + return camelName; + } + +public: + using TPtr = TIntrusivePtr<TFieldGenerator>; + + explicit TFieldGenerator(const FieldDescriptor* field, const TString& className) + : Field(field) + , Vars({ + {"class", className}, + {"func", "validate_" + field->name()}, + {"field", field->name()}, + {"PascalName", PascalName(field->camelcase_name())}, + }) + { + } + + const TVariables& GetVars() const { + return Vars; + } + + void Declare(TPrinter& printer) const { + if (!HasValidators() && !IsCustomMessage(Field->message_type())) { + printer->Print(Vars, "bool $func$(TProtoStringType&) const { return true; }\n"); + } else { + printer->Print(Vars, "bool $func$(TProtoStringType& __err) const;\n"); + } + } + + void Define(TPrinter& printer) const { + if (!HasValidators() && !IsCustomMessage(Field->message_type())) { + return; + } + + printer->Print(Vars, "bool $class$::$func$(TProtoStringType& __err) const {\n"); + printer->Indent(); + Body(printer); + printer->Outdent(); + printer->Print(Vars, "}\n\n"); + } + +private: + const FieldDescriptor* Field; + const TVariables Vars; + +}; // TFieldGenerator + +class TOneofGenerator: public TThrRefBase { + void Body(TPrinter& printer) const { + printer->Print(Vars, "switch ($oneof$_case()) {\n"); + + for (auto field : Fields) { + printer->Print(field->GetVars(), "case k$PascalName$:\n"); + printer->Indent(); + printer->Print(field->GetVars(), "return $func$(__err);\n"); + printer->Outdent(); + } + + printer->Print(Vars, "default:\n"); + printer->Indent(); + printer->Print(Vars, "return true;\n"); + printer->Outdent(); + + printer->Print(Vars, "}\n"); + } + +public: + using TPtr = TIntrusivePtr<TOneofGenerator>; + + explicit TOneofGenerator(const OneofDescriptor* oneof, const TString& className) + : Vars({ + {"class", className}, + {"oneof", oneof->name()}, + {"func", "validate_" + oneof->name()}, + }) + { + } + + const TVariables& GetVars() const { + return Vars; + } + + void AddField(TFieldGenerator::TPtr field) { + Fields.push_back(field); + } + + void Declare(TPrinter& printer) const { + printer->Print(Vars, "bool $func$(TProtoStringType& __err) const;\n"); + } + + void Define(TPrinter& printer) const { + printer->Print(Vars, "bool $class$::$func$(TProtoStringType& __err) const {\n"); + printer->Indent(); + Body(printer); + printer->Outdent(); + printer->Print(Vars, "}\n\n"); + } + +private: + const TVariables Vars; + TVector<TFieldGenerator::TPtr> Fields; + +}; // TOneofGenerator + +class TMessageGenerator { + struct TItem { + TOneofGenerator::TPtr Oneof; + TVector<TFieldGenerator::TPtr> Fields; + + explicit TItem(TOneofGenerator::TPtr oneof) + : Oneof(oneof) + { + } + + void AddField(TFieldGenerator::TPtr field) { + Fields.push_back(field); + if (Oneof) { + Oneof->AddField(field); + } + } + }; + + using TItems = THashMap<const OneofDescriptor*, TItem>; + + void Declare(TPrinter& printer, const TItems& items) { + if (!items) { + printer->Print(Vars, "bool validate(TProtoStringType&) const { return true; }\n"); + } else { + printer->Print(Vars, "bool validate(TProtoStringType& __err) const;\n"); + } + } + + void Define(TPrinter& printer, const TItems& items) { + if (!items) { + return; + } + + printer->Print(Vars, "bool $class$::validate(TProtoStringType& __err) const {\n"); + printer->Indent(); + printer->Print(Vars, "return\n"); + printer->Indent(); + + bool first = true; + for (const auto& [_, item] : items) { + if (!item.Oneof) { + for (const auto& field : item.Fields) { + if (first) { + first = false; + } else { + printer->Print(Vars, "&& "); + } + printer->Print(field->GetVars(), "$func$(__err)\n"); + } + } else { + if (first) { + first = false; + } else { + printer->Print(Vars, "&& "); + } + printer->Print(item.Oneof->GetVars(), "$func$(__err)\n"); + } + } + + printer->Outdent(); + printer->Print(Vars, ";\n"); + printer->Outdent(); + printer->Print(Vars, "}\n\n"); + } + +public: + explicit TMessageGenerator(const Descriptor* message, OutputDirectory* output) + : Message(message) + , Output(output) + , Header(output, HeaderFileName(message), ClassScope(message)) + , Source(output, SourceFileName(message), NamespaceScope()) + , Vars({ + {"class", ClassName(message)}, + }) + { + } + + void Generate() { + for (auto i = 0; i < Message->nested_type_count(); ++i) { + const Descriptor* message = Message->nested_type(i); + if (!IsCustomMessage(message)) { + continue; + } + + TMessageGenerator mg(message, Output); + mg.Generate(); + } + + TItems items; + + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + const OneofDescriptor* oneof = field->containing_oneof(); + + TFieldGenerator::TPtr fieldGen = new TFieldGenerator(field, Vars.at("class")); + TOneofGenerator::TPtr oneofGen = oneof ? new TOneofGenerator(oneof, Vars.at("class")) : nullptr; + + auto it = items.find(oneof); + if (it == items.end()) { + it = items.emplace(oneof, TItem(oneofGen)).first; + } + + it->second.AddField(fieldGen); + } + + for (const auto& [_, item] : items) { + for (const auto& field : item.Fields) { + field->Declare(Header); + field->Define(Source); + } + + if (item.Oneof) { + item.Oneof->Declare(Header); + item.Oneof->Define(Source); + } + } + + Declare(Header, items); + Define(Source, items); + } + +private: + const Descriptor* Message; + OutputDirectory* Output; + TPrinter Header; + TPrinter Source; + const TVariables Vars; + +}; // TMessageGenerator + +class TCodeGenerator: public CodeGenerator { + bool Generate( + const FileDescriptor* file, + const TProtoStringType&, + OutputDirectory* output, + TProtoStringType*) const override final { + + for (auto i = 0; i < file->message_type_count(); ++i) { + TMessageGenerator mg(file->message_type(i), output); + mg.Generate(); + } + + return true; + } + +}; // TCodeGenerator + +} // NValidation +} // NKikimr + +int main(int argc, char* argv[]) { + NKikimr::NValidation::TCodeGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/ydb/core/grpc_services/validation/ut.cpp b/ydb/core/grpc_services/validation/ut.cpp index 0912e08c267..ca414df810f 100644 --- a/ydb/core/grpc_services/validation/ut.cpp +++ b/ydb/core/grpc_services/validation/ut.cpp @@ -1,288 +1,288 @@ #include <ydb/core/grpc_services/validation/ut/protos/validation_test.pb.h> - + #include <library/cpp/testing/unittest/registar.h> - -Y_UNIT_TEST_SUITE(ValidationTests) { - using namespace Ydb::ValidationTest; - - template <typename T> - bool Validate(const T& proto, TString& error) { - return proto.validate(error); - } - - template <typename T> - bool Validate(const T& proto) { - TString unused; - return Validate(proto, unused); - } - - template <typename T> - void Validate(const T& proto, const TString& errorContains) { - TString error; - UNIT_ASSERT(!Validate(proto, error)); - UNIT_ASSERT_STRING_CONTAINS(error, errorContains); - } - - Y_UNIT_TEST(Empty) { - UNIT_ASSERT(Validate(EmptyMessage())); - } - - Y_UNIT_TEST(WithouValidators) { - UNIT_ASSERT(Validate(MessageWithoutValidators())); - } - - Y_UNIT_TEST(Basic) { - MessageBasic basic; - - // required - { - Validate(basic, "required but not set"); - basic.set_check_required("set"); - } - - // size: range - { - Validate(basic, "size is not in"); - basic.mutable_check_size_range_1_2()->Add("string"); - basic.mutable_check_size_range_1_2()->Add("string"); - basic.mutable_check_size_range_1_2()->Add("string"); // extra - - Validate(basic, "size is not in"); - basic.mutable_check_size_range_1_2()->RemoveLast(); - } - - // size: lt - { - basic.mutable_check_size_lt_2()->Add("1"); - basic.mutable_check_size_lt_2()->Add("2"); // extra - Validate(basic, "size is not <"); - basic.mutable_check_size_lt_2()->RemoveLast(); - } - - // size: le - { - basic.mutable_check_size_le_1()->Add("1"); - basic.mutable_check_size_le_1()->Add("2"); // extra - Validate(basic, "size is not <="); - basic.mutable_check_size_le_1()->RemoveLast(); - } - - // size: eq - { - basic.mutable_check_size_eq_1()->Add("1"); - basic.mutable_check_size_eq_1()->Add("2"); // extra - Validate(basic, "size is not =="); - basic.mutable_check_size_eq_1()->RemoveLast(); - } - - // size: ge - { - Validate(basic, "size is not >="); - basic.mutable_check_size_ge_1()->Add("1"); - } - - // size: gt - { - basic.mutable_check_size_gt_1()->Add("1"); - Validate(basic, "size is not >"); - basic.mutable_check_size_gt_1()->Add("1"); - } - - // length: range - { - Validate(basic, "length is not in"); - basic.set_check_length_range_1_3("value"); - - Validate(basic, "length is not in"); - basic.set_check_length_range_1_3("val"); - } - - // length: lt - { - basic.set_check_length_lt_4("value"); - Validate(basic, "length is not <"); - basic.set_check_length_lt_4("val"); - } - - // length: le - { - basic.set_check_length_le_3("value"); - Validate(basic, "length is not <="); - basic.set_check_length_le_3("val"); - } - - // length: eq - { - basic.set_check_length_eq_3("value"); - Validate(basic, "length is not =="); - basic.set_check_length_eq_3("val"); - } - - // length: ge - { - basic.set_check_length_ge_3("v"); - Validate(basic, "length is not >="); - basic.set_check_length_ge_3("val"); - } - - // length: gt - { - basic.set_check_length_gt_3("val"); - Validate(basic, "length is not >"); - basic.set_check_length_gt_3("value"); - } - - // value: int32 - { - basic.set_check_value_int32(-1); - Validate(basic, "value is not"); - basic.set_check_value_int32(-2); - } - - // value: int64 - { - basic.set_check_value_int64(-5'000'000'000); - Validate(basic, "value is not"); - basic.set_check_value_int64(-5'000'000'001); - } - - // value: uint32 - { - basic.set_check_value_uint32(1); - Validate(basic, "value is not"); - basic.set_check_value_uint32(2); - } - - // value: uint64 - { - basic.set_check_value_uint64(5'000'000'000); - Validate(basic, "value is not"); - basic.set_check_value_uint64(5'000'000'001); - } - - // value: double - { - basic.set_check_value_double(0.99); - Validate(basic, "value is not"); - basic.set_check_value_double(1.01); - } - - // value: float - { - basic.set_check_value_float(-0.99); - Validate(basic, "value is not"); - basic.set_check_value_float(-1.01); - } - - // value: bool - { - basic.set_check_value_bool(false); - Validate(basic, "value is not"); - basic.set_check_value_bool(true); - } - - // value: string - { - basic.set_check_value_string("changeme"); - Validate(basic, "value is not"); - basic.set_check_value_string("changed"); - } - - // value: range [1;10] - { - basic.set_check_value_range_1(11); - Validate(basic, "value is not"); - basic.set_check_value_range_1(10); - } - - // value: range (1;10] - { - basic.set_check_value_range_2(1); - Validate(basic, "value is not"); - basic.set_check_value_range_2(2); - } - - // value: range [1;10) - { - basic.set_check_value_range_3(10); - Validate(basic, "value is not"); - basic.set_check_value_range_3(9); - } - - // value: range (1;10) - { - basic.set_check_value_range_4(1); - Validate(basic, "value is not"); - basic.set_check_value_range_4(5); - } - - UNIT_ASSERT(Validate(basic)); - } - - Y_UNIT_TEST(Map) { - MessageMap map; - - // size - { - Validate(map, "size is not >"); - map.mutable_values()->insert({"key", "value"}); - } - - // value's length - { - Validate(map, "value length is not <"); - map.mutable_values()->at("key") = "v"; - } - - // key's length - { - Validate(map, "key length is not <"); - map.mutable_values()->erase("key"); - map.mutable_values()->insert({"k", "v"}); - } - - // key - { - Validate(map, "key is not"); - map.mutable_values()->erase("k"); - map.mutable_values()->insert({"a", "v"}); - } - - // value - { - Validate(map, "value is not"); - map.mutable_values()->erase("a"); - map.mutable_values()->insert({"a", "b"}); - } - - UNIT_ASSERT(Validate(map)); - } - - Y_UNIT_TEST(Oneof) { - MessageOneof oneof; - - // required - { - Validate(oneof, "required but not set"); - oneof.set_check_required("set"); - } - - // check oneof (1) - { - oneof.set_check_le_1("12"); - Validate(oneof, "length is not <="); - - oneof.set_check_le_1("1"); - UNIT_ASSERT(Validate(oneof)); - } - - // check oneof (2) - { - oneof.set_check_le_2("123"); - Validate(oneof, "length is not <="); - - oneof.set_check_le_2("12"); - UNIT_ASSERT(Validate(oneof)); - } - } -} + +Y_UNIT_TEST_SUITE(ValidationTests) { + using namespace Ydb::ValidationTest; + + template <typename T> + bool Validate(const T& proto, TString& error) { + return proto.validate(error); + } + + template <typename T> + bool Validate(const T& proto) { + TString unused; + return Validate(proto, unused); + } + + template <typename T> + void Validate(const T& proto, const TString& errorContains) { + TString error; + UNIT_ASSERT(!Validate(proto, error)); + UNIT_ASSERT_STRING_CONTAINS(error, errorContains); + } + + Y_UNIT_TEST(Empty) { + UNIT_ASSERT(Validate(EmptyMessage())); + } + + Y_UNIT_TEST(WithouValidators) { + UNIT_ASSERT(Validate(MessageWithoutValidators())); + } + + Y_UNIT_TEST(Basic) { + MessageBasic basic; + + // required + { + Validate(basic, "required but not set"); + basic.set_check_required("set"); + } + + // size: range + { + Validate(basic, "size is not in"); + basic.mutable_check_size_range_1_2()->Add("string"); + basic.mutable_check_size_range_1_2()->Add("string"); + basic.mutable_check_size_range_1_2()->Add("string"); // extra + + Validate(basic, "size is not in"); + basic.mutable_check_size_range_1_2()->RemoveLast(); + } + + // size: lt + { + basic.mutable_check_size_lt_2()->Add("1"); + basic.mutable_check_size_lt_2()->Add("2"); // extra + Validate(basic, "size is not <"); + basic.mutable_check_size_lt_2()->RemoveLast(); + } + + // size: le + { + basic.mutable_check_size_le_1()->Add("1"); + basic.mutable_check_size_le_1()->Add("2"); // extra + Validate(basic, "size is not <="); + basic.mutable_check_size_le_1()->RemoveLast(); + } + + // size: eq + { + basic.mutable_check_size_eq_1()->Add("1"); + basic.mutable_check_size_eq_1()->Add("2"); // extra + Validate(basic, "size is not =="); + basic.mutable_check_size_eq_1()->RemoveLast(); + } + + // size: ge + { + Validate(basic, "size is not >="); + basic.mutable_check_size_ge_1()->Add("1"); + } + + // size: gt + { + basic.mutable_check_size_gt_1()->Add("1"); + Validate(basic, "size is not >"); + basic.mutable_check_size_gt_1()->Add("1"); + } + + // length: range + { + Validate(basic, "length is not in"); + basic.set_check_length_range_1_3("value"); + + Validate(basic, "length is not in"); + basic.set_check_length_range_1_3("val"); + } + + // length: lt + { + basic.set_check_length_lt_4("value"); + Validate(basic, "length is not <"); + basic.set_check_length_lt_4("val"); + } + + // length: le + { + basic.set_check_length_le_3("value"); + Validate(basic, "length is not <="); + basic.set_check_length_le_3("val"); + } + + // length: eq + { + basic.set_check_length_eq_3("value"); + Validate(basic, "length is not =="); + basic.set_check_length_eq_3("val"); + } + + // length: ge + { + basic.set_check_length_ge_3("v"); + Validate(basic, "length is not >="); + basic.set_check_length_ge_3("val"); + } + + // length: gt + { + basic.set_check_length_gt_3("val"); + Validate(basic, "length is not >"); + basic.set_check_length_gt_3("value"); + } + + // value: int32 + { + basic.set_check_value_int32(-1); + Validate(basic, "value is not"); + basic.set_check_value_int32(-2); + } + + // value: int64 + { + basic.set_check_value_int64(-5'000'000'000); + Validate(basic, "value is not"); + basic.set_check_value_int64(-5'000'000'001); + } + + // value: uint32 + { + basic.set_check_value_uint32(1); + Validate(basic, "value is not"); + basic.set_check_value_uint32(2); + } + + // value: uint64 + { + basic.set_check_value_uint64(5'000'000'000); + Validate(basic, "value is not"); + basic.set_check_value_uint64(5'000'000'001); + } + + // value: double + { + basic.set_check_value_double(0.99); + Validate(basic, "value is not"); + basic.set_check_value_double(1.01); + } + + // value: float + { + basic.set_check_value_float(-0.99); + Validate(basic, "value is not"); + basic.set_check_value_float(-1.01); + } + + // value: bool + { + basic.set_check_value_bool(false); + Validate(basic, "value is not"); + basic.set_check_value_bool(true); + } + + // value: string + { + basic.set_check_value_string("changeme"); + Validate(basic, "value is not"); + basic.set_check_value_string("changed"); + } + + // value: range [1;10] + { + basic.set_check_value_range_1(11); + Validate(basic, "value is not"); + basic.set_check_value_range_1(10); + } + + // value: range (1;10] + { + basic.set_check_value_range_2(1); + Validate(basic, "value is not"); + basic.set_check_value_range_2(2); + } + + // value: range [1;10) + { + basic.set_check_value_range_3(10); + Validate(basic, "value is not"); + basic.set_check_value_range_3(9); + } + + // value: range (1;10) + { + basic.set_check_value_range_4(1); + Validate(basic, "value is not"); + basic.set_check_value_range_4(5); + } + + UNIT_ASSERT(Validate(basic)); + } + + Y_UNIT_TEST(Map) { + MessageMap map; + + // size + { + Validate(map, "size is not >"); + map.mutable_values()->insert({"key", "value"}); + } + + // value's length + { + Validate(map, "value length is not <"); + map.mutable_values()->at("key") = "v"; + } + + // key's length + { + Validate(map, "key length is not <"); + map.mutable_values()->erase("key"); + map.mutable_values()->insert({"k", "v"}); + } + + // key + { + Validate(map, "key is not"); + map.mutable_values()->erase("k"); + map.mutable_values()->insert({"a", "v"}); + } + + // value + { + Validate(map, "value is not"); + map.mutable_values()->erase("a"); + map.mutable_values()->insert({"a", "b"}); + } + + UNIT_ASSERT(Validate(map)); + } + + Y_UNIT_TEST(Oneof) { + MessageOneof oneof; + + // required + { + Validate(oneof, "required but not set"); + oneof.set_check_required("set"); + } + + // check oneof (1) + { + oneof.set_check_le_1("12"); + Validate(oneof, "length is not <="); + + oneof.set_check_le_1("1"); + UNIT_ASSERT(Validate(oneof)); + } + + // check oneof (2) + { + oneof.set_check_le_2("123"); + Validate(oneof, "length is not <="); + + oneof.set_check_le_2("12"); + UNIT_ASSERT(Validate(oneof)); + } + } +} diff --git a/ydb/core/grpc_services/validation/ut/protos/validation_test.proto b/ydb/core/grpc_services/validation/ut/protos/validation_test.proto index 6e2b1dcc896..f2c26ab3887 100644 --- a/ydb/core/grpc_services/validation/ut/protos/validation_test.proto +++ b/ydb/core/grpc_services/validation/ut/protos/validation_test.proto @@ -1,60 +1,60 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/annotations/validation.proto"; - -package Ydb.ValidationTest; - -message EmptyMessage { -} - -message MessageWithoutValidators { - uint32 key = 1; - string value = 2; - repeated string payload = 3; -} - -message MessageBasic { - string check_required = 1 [(required) = true]; - - repeated string check_size_range_1_2 = 2 [(size).range = {min: 1, max: 2}]; - repeated string check_size_lt_2 = 3 [(size).lt = 2]; - repeated string check_size_le_1 = 4 [(size).le = 1]; - repeated string check_size_eq_1 = 5 [(size).eq = 1]; - repeated string check_size_ge_1 = 6 [(size).ge = 1]; - repeated string check_size_gt_1 = 7 [(size).gt = 1]; - - string check_length_range_1_3 = 8 [(length).range = {min: 1, max: 3}]; - string check_length_lt_4 = 9 [(length).lt = 4]; - string check_length_le_3 = 10 [(length).le = 3]; - string check_length_eq_3 = 11 [(length).eq = 3]; - string check_length_ge_3 = 12 [(length).ge = 3]; - string check_length_gt_3 = 13 [(length).gt = 3]; - - int32 check_value_int32 = 14 [(value) = "< -1"]; - int64 check_value_int64 = 15 [(value) = "< -5'000'000'000"]; - uint32 check_value_uint32 = 16 [(value) = "> 1"]; - uint64 check_value_uint64 = 17 [(value) = "> 5'000'000'000"]; - double check_value_double = 18 [(value) = "> 1.0"]; - float check_value_float = 19 [(value) = "< -1.0"]; - bool check_value_bool = 20 [(value) = "== true"]; - string check_value_string = 21 [(value) = '!= "changeme"']; - - int32 check_value_range_1 = 22 [(value) = "[1; 10]"]; - int32 check_value_range_2 = 23 [(value) = "(1; 10]"]; - int32 check_value_range_3 = 24 [(value) = "[1; 10)"]; - int32 check_value_range_4 = 25 [(value) = "(1; 10)"]; -} - -message MessageMap { - map<string, string> values = 1 [(size).gt = 0, (map_key).length.lt = 2, (length).lt = 2, (map_key).value = '!= "k"', (value) = '!= "v"']; -} - -message MessageOneof { - uint32 key = 1; - string check_required = 2 [(required) = true]; - oneof value { - string check_le_1 = 3 [(length).le = 1]; - string check_le_2 = 4 [(length).le = 2]; - } -} + +package Ydb.ValidationTest; + +message EmptyMessage { +} + +message MessageWithoutValidators { + uint32 key = 1; + string value = 2; + repeated string payload = 3; +} + +message MessageBasic { + string check_required = 1 [(required) = true]; + + repeated string check_size_range_1_2 = 2 [(size).range = {min: 1, max: 2}]; + repeated string check_size_lt_2 = 3 [(size).lt = 2]; + repeated string check_size_le_1 = 4 [(size).le = 1]; + repeated string check_size_eq_1 = 5 [(size).eq = 1]; + repeated string check_size_ge_1 = 6 [(size).ge = 1]; + repeated string check_size_gt_1 = 7 [(size).gt = 1]; + + string check_length_range_1_3 = 8 [(length).range = {min: 1, max: 3}]; + string check_length_lt_4 = 9 [(length).lt = 4]; + string check_length_le_3 = 10 [(length).le = 3]; + string check_length_eq_3 = 11 [(length).eq = 3]; + string check_length_ge_3 = 12 [(length).ge = 3]; + string check_length_gt_3 = 13 [(length).gt = 3]; + + int32 check_value_int32 = 14 [(value) = "< -1"]; + int64 check_value_int64 = 15 [(value) = "< -5'000'000'000"]; + uint32 check_value_uint32 = 16 [(value) = "> 1"]; + uint64 check_value_uint64 = 17 [(value) = "> 5'000'000'000"]; + double check_value_double = 18 [(value) = "> 1.0"]; + float check_value_float = 19 [(value) = "< -1.0"]; + bool check_value_bool = 20 [(value) = "== true"]; + string check_value_string = 21 [(value) = '!= "changeme"']; + + int32 check_value_range_1 = 22 [(value) = "[1; 10]"]; + int32 check_value_range_2 = 23 [(value) = "(1; 10]"]; + int32 check_value_range_3 = 24 [(value) = "[1; 10)"]; + int32 check_value_range_4 = 25 [(value) = "(1; 10)"]; +} + +message MessageMap { + map<string, string> values = 1 [(size).gt = 0, (map_key).length.lt = 2, (length).lt = 2, (map_key).value = '!= "k"', (value) = '!= "v"']; +} + +message MessageOneof { + uint32 key = 1; + string check_required = 2 [(required) = true]; + oneof value { + string check_le_1 = 3 [(length).le = 1]; + string check_le_2 = 4 [(length).le = 2]; + } +} diff --git a/ydb/core/grpc_services/validation/ut/protos/ya.make b/ydb/core/grpc_services/validation/ut/protos/ya.make index 4f02d06acb2..366c4a43399 100644 --- a/ydb/core/grpc_services/validation/ut/protos/ya.make +++ b/ydb/core/grpc_services/validation/ut/protos/ya.make @@ -1,20 +1,20 @@ -PROTO_LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( +PROTO_LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( ydb/public/api/protos/annotations -) - -SRCS( - validation_test.proto -) - +) + +SRCS( + validation_test.proto +) + CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation) - + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/core/grpc_services/validation/ut/ya.make b/ydb/core/grpc_services/validation/ut/ya.make index e03b75e2438..a3eb1187805 100644 --- a/ydb/core/grpc_services/validation/ut/ya.make +++ b/ydb/core/grpc_services/validation/ut/ya.make @@ -1,23 +1,23 @@ UNITTEST_FOR(ydb/core/grpc_services/validation) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( library/cpp/testing/unittest ydb/core/grpc_services/validation/ut/protos -) - -SRCS( - ut.cpp -) - -END() +) + +SRCS( + ut.cpp +) + +END() diff --git a/ydb/core/grpc_services/validation/ya.make b/ydb/core/grpc_services/validation/ya.make index b6781a5c3b8..a0efda9559c 100644 --- a/ydb/core/grpc_services/validation/ya.make +++ b/ydb/core/grpc_services/validation/ya.make @@ -1,21 +1,21 @@ -PROGRAM() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( +PROGRAM() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( contrib/libs/protoc ydb/public/api/protos/annotations -) - -SRCS( - helpers.cpp - main.cpp -) - -END() +) + +SRCS( + helpers.cpp + main.cpp +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 91f5a5f106b..05b156bf043 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -18,12 +18,12 @@ SRCS( rpc_alter_table.cpp rpc_begin_transaction.cpp rpc_calls.cpp - rpc_cancel_operation.cpp + rpc_cancel_operation.cpp rpc_cms.cpp rpc_commit_transaction.cpp rpc_copy_table.cpp rpc_copy_tables.cpp - rpc_export.cpp + rpc_export.cpp rpc_create_coordination_node.cpp rpc_create_session.cpp rpc_create_table.cpp @@ -40,16 +40,16 @@ SRCS( rpc_execute_yql_script.cpp rpc_explain_yql_script.cpp rpc_explain_data_query.cpp - rpc_forget_operation.cpp + rpc_forget_operation.cpp rpc_get_operation.cpp rpc_get_shard_locations.cpp - rpc_import.cpp - rpc_import_data.cpp + rpc_import.cpp + rpc_import_data.cpp rpc_keep_alive.cpp rpc_kh_describe.cpp rpc_kh_snapshots.cpp rpc_kqp_base.cpp - rpc_list_operations.cpp + rpc_list_operations.cpp rpc_login.cpp rpc_load_rows.cpp rpc_log_store.cpp diff --git a/ydb/core/io_formats/csv.h b/ydb/core/io_formats/csv.h index a3d9547b8c9..d3e75a487df 100644 --- a/ydb/core/io_formats/csv.h +++ b/ydb/core/io_formats/csv.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> - -#include <util/generic/strbuf.h> -#include <util/memory/pool.h> - + +#include <util/generic/strbuf.h> +#include <util/memory/pool.h> + #include <contrib/libs/apache/arrow/cpp/src/arrow/csv/api.h> #include <contrib/libs/apache/arrow/cpp/src/arrow/io/api.h> namespace NKikimr::NFormats { - + struct TYdbDump { // Parse YdbDump-formatted line static bool ParseLine(TStringBuf line, const std::vector<std::pair<i32, ui32>>& columnOrderTypes, TMemoryPool& pool, TVector<TCell>& keys, TVector<TCell>& values, TString& strError, ui64& numBytes); }; - + class TArrowCSV { public: static constexpr ui32 DEFAULT_BLOCK_SIZE = 1024 * 1024; diff --git a/ydb/core/io_formats/csv_ydb_dump.cpp b/ydb/core/io_formats/csv_ydb_dump.cpp index f5a602df4cd..803c4eb531a 100644 --- a/ydb/core/io_formats/csv_ydb_dump.cpp +++ b/ydb/core/io_formats/csv_ydb_dump.cpp @@ -1,267 +1,267 @@ #include "csv.h" - -#include <contrib/libs/double-conversion/double-conversion.h> - + +#include <contrib/libs/double-conversion/double-conversion.h> + #include <ydb/library/binary_json/write.h> #include <ydb/library/dynumber/dynumber.h> - -#include <library/cpp/string_utils/quote/quote.h> - + +#include <library/cpp/string_utils/quote/quote.h> + #include <ydb/library/yql/minikql/dom/yson.h> #include <ydb/library/yql/minikql/dom/json.h> #include <ydb/library/yql/public/udf/udf_types.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> #include <ydb/library/yql/utils/utf8.h> - -#include <util/datetime/base.h> -#include <util/string/cast.h> - + +#include <util/datetime/base.h> +#include <util/string/cast.h> + namespace NKikimr::NFormats { - -namespace { - - bool CheckedUnescape(TStringBuf value, TString& result) { - if (value.size() < 2 || value.front() != '"' || value.back() != '"') { - return false; - } - - result = CGIUnescapeRet(value.Skip(1).Chop(1)); - return true; - } - - template <typename T> - bool TryParse(TStringBuf value, T& result) { - return TryFromString(value, result); - } - - template <> - bool TryParse(TStringBuf value, double& result) { - struct TCvt: public double_conversion::StringToDoubleConverter { - inline TCvt() - : StringToDoubleConverter(ALLOW_TRAILING_JUNK | ALLOW_HEX | ALLOW_LEADING_SPACES, 0.0, NAN, "inf", "nan") - { - } - }; - - int processed = 0; - result = Singleton<TCvt>()->StringToDouble(value.Data(), value.Size(), &processed); - - return static_cast<size_t>(processed) == value.Size(); - } - - template <> - bool TryParse(TStringBuf value, float& result) { - double tmp; - if (TryParse(value, tmp)) { - result = static_cast<float>(tmp); - return true; - } - - return false; - } - - template <> - bool TryParse(TStringBuf value, TInstant& result) { - return TInstant::TryParseIso8601(value, result); - } - - template <> - bool TryParse(TStringBuf value, TString& result) { - return CheckedUnescape(value, result); - } - - template <> - bool TryParse(TStringBuf value, NYql::NDecimal::TInt128& result) { - if (!NYql::NDecimal::IsValid(value)) { - return false; - } - - result = NYql::NDecimal::FromString(value, NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE); - return true; - } - - template <> - bool TryParse(TStringBuf value, TMaybe<NBinaryJson::TBinaryJson>& result) { - TString unescaped; - if (!CheckedUnescape(value, unescaped)) { - return false; - } - - result = NBinaryJson::SerializeToBinaryJson(unescaped); - return result.Defined(); - } - - template <> - bool TryParse(TStringBuf value, TMaybe<TString>& result) { - result = NDyNumber::ParseDyNumberString(value); - return result.Defined(); - } - - template <typename T, typename U> - using TConverter = std::function<U(const T&)>; - - template <typename T, typename U> - U Implicit(const T& v) { - return v; - } - - ui16 Days(const TInstant& v) { - return v.Days(); - } - - ui32 Seconds(const TInstant& v) { - return v.Seconds(); - } - - ui64 MicroSeconds(const TInstant& v) { - return v.MicroSeconds(); - } - - std::pair<ui64, ui64> Int128ToPair(const NYql::NDecimal::TInt128& v) { - return NYql::NDecimal::MakePair(v); - } - - TStringBuf BinaryJsonToStringBuf(const TMaybe<NBinaryJson::TBinaryJson>& v) { - Y_VERIFY(v.Defined()); - return TStringBuf(v->Data(), v->Size()); - } - - TStringBuf DyNumberToStringBuf(const TMaybe<TString>& v) { - Y_VERIFY(v.Defined()); - return TStringBuf(*v); - } - - template <typename T, typename U = T> - struct TCellMaker { - static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, U> conv = &Implicit<T, U>) { - T t; - if (!TryParse<T>(v, t)) { - return false; - } - - auto& u = *pool.Allocate<U>(); - u = conv(t); - c = TCell(reinterpret_cast<const char*>(&u), sizeof(u)); - - return true; - } - }; - - template <typename T> - struct TCellMaker<T, TStringBuf> { - static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, TStringBuf> conv = &Implicit<T, TStringBuf>) { - T t; - if (!TryParse<T>(v, t)) { - return false; - } - - const auto u = pool.AppendString(conv(t)); - c = TCell(u.data(), u.size()); - - return true; - } - }; - -} // anonymous - -bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeId type, TMemoryPool& pool) { - if (value == "null") { - return true; - } - - switch (type) { - case NScheme::NTypeIds::Bool: - return TCellMaker<bool>::Make(cell, value, pool); - case NScheme::NTypeIds::Int8: - return TCellMaker<i8>::Make(cell, value, pool); - case NScheme::NTypeIds::Uint8: - return TCellMaker<ui8>::Make(cell, value, pool); - case NScheme::NTypeIds::Int16: - return TCellMaker<i16>::Make(cell, value, pool); - case NScheme::NTypeIds::Uint16: - return TCellMaker<ui16>::Make(cell, value, pool); - case NScheme::NTypeIds::Int32: - return TCellMaker<i32>::Make(cell, value, pool); - case NScheme::NTypeIds::Uint32: - return TCellMaker<ui32>::Make(cell, value, pool); - case NScheme::NTypeIds::Int64: - return TCellMaker<i64>::Make(cell, value, pool); - case NScheme::NTypeIds::Uint64: - return TCellMaker<ui64>::Make(cell, value, pool); - case NScheme::NTypeIds::Float: - return TCellMaker<float>::Make(cell, value, pool); - case NScheme::NTypeIds::Double: - return TCellMaker<double>::Make(cell, value, pool); - case NScheme::NTypeIds::Date: - return TCellMaker<TInstant, ui16>::Make(cell, value, pool, &Days); - case NScheme::NTypeIds::Datetime: - return TCellMaker<TInstant, ui32>::Make(cell, value, pool, &Seconds); - case NScheme::NTypeIds::Timestamp: - return TCellMaker<TInstant, ui64>::Make(cell, value, pool, &MicroSeconds); - case NScheme::NTypeIds::Interval: - return TCellMaker<i64>::Make(cell, value, pool); - case NScheme::NTypeIds::String: - case NScheme::NTypeIds::String4k: - case NScheme::NTypeIds::String2m: - case NScheme::NTypeIds::Utf8: - case NScheme::NTypeIds::Yson: - case NScheme::NTypeIds::Json: - return TCellMaker<TString, TStringBuf>::Make(cell, value, pool); - case NScheme::NTypeIds::JsonDocument: - return TCellMaker<TMaybe<NBinaryJson::TBinaryJson>, TStringBuf>::Make(cell, value, pool, &BinaryJsonToStringBuf); - case NScheme::NTypeIds::DyNumber: - return TCellMaker<TMaybe<TString>, TStringBuf>::Make(cell, value, pool, &DyNumberToStringBuf); - case NScheme::NTypeIds::Decimal: - return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, &Int128ToPair); - default: - return false; - } -} - -bool CheckCellValue(const TCell& cell, NScheme::TTypeId type) { - if (cell.IsNull()) { - return true; - } - - switch (type) { - case NScheme::NTypeIds::Bool: - case NScheme::NTypeIds::Int8: - case NScheme::NTypeIds::Uint8: - case NScheme::NTypeIds::Int16: - case NScheme::NTypeIds::Uint16: - case NScheme::NTypeIds::Int32: - case NScheme::NTypeIds::Uint32: - case NScheme::NTypeIds::Int64: - case NScheme::NTypeIds::Uint64: - case NScheme::NTypeIds::Float: - case NScheme::NTypeIds::Double: - case NScheme::NTypeIds::String: - case NScheme::NTypeIds::String4k: - case NScheme::NTypeIds::String2m: - case NScheme::NTypeIds::JsonDocument: // checked at parsing time - case NScheme::NTypeIds::DyNumber: // checked at parsing time - return true; - case NScheme::NTypeIds::Date: - return cell.AsValue<ui16>() < NUdf::MAX_DATE; - case NScheme::NTypeIds::Datetime: - return cell.AsValue<ui32>() < NUdf::MAX_DATETIME; - case NScheme::NTypeIds::Timestamp: - return cell.AsValue<ui64>() < NUdf::MAX_TIMESTAMP; - case NScheme::NTypeIds::Interval: - return (ui64)std::abs(cell.AsValue<i64>()) < NUdf::MAX_TIMESTAMP; - case NScheme::NTypeIds::Utf8: - return NYql::IsUtf8(cell.AsBuf()); - case NScheme::NTypeIds::Yson: - return NYql::NDom::IsValidYson(cell.AsBuf()); - case NScheme::NTypeIds::Json: - return NYql::NDom::IsValidJson(cell.AsBuf()); - case NScheme::NTypeIds::Decimal: - return !NYql::NDecimal::IsError(cell.AsValue<NYql::NDecimal::TInt128>()); - default: - return false; - } -} - + +namespace { + + bool CheckedUnescape(TStringBuf value, TString& result) { + if (value.size() < 2 || value.front() != '"' || value.back() != '"') { + return false; + } + + result = CGIUnescapeRet(value.Skip(1).Chop(1)); + return true; + } + + template <typename T> + bool TryParse(TStringBuf value, T& result) { + return TryFromString(value, result); + } + + template <> + bool TryParse(TStringBuf value, double& result) { + struct TCvt: public double_conversion::StringToDoubleConverter { + inline TCvt() + : StringToDoubleConverter(ALLOW_TRAILING_JUNK | ALLOW_HEX | ALLOW_LEADING_SPACES, 0.0, NAN, "inf", "nan") + { + } + }; + + int processed = 0; + result = Singleton<TCvt>()->StringToDouble(value.Data(), value.Size(), &processed); + + return static_cast<size_t>(processed) == value.Size(); + } + + template <> + bool TryParse(TStringBuf value, float& result) { + double tmp; + if (TryParse(value, tmp)) { + result = static_cast<float>(tmp); + return true; + } + + return false; + } + + template <> + bool TryParse(TStringBuf value, TInstant& result) { + return TInstant::TryParseIso8601(value, result); + } + + template <> + bool TryParse(TStringBuf value, TString& result) { + return CheckedUnescape(value, result); + } + + template <> + bool TryParse(TStringBuf value, NYql::NDecimal::TInt128& result) { + if (!NYql::NDecimal::IsValid(value)) { + return false; + } + + result = NYql::NDecimal::FromString(value, NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE); + return true; + } + + template <> + bool TryParse(TStringBuf value, TMaybe<NBinaryJson::TBinaryJson>& result) { + TString unescaped; + if (!CheckedUnescape(value, unescaped)) { + return false; + } + + result = NBinaryJson::SerializeToBinaryJson(unescaped); + return result.Defined(); + } + + template <> + bool TryParse(TStringBuf value, TMaybe<TString>& result) { + result = NDyNumber::ParseDyNumberString(value); + return result.Defined(); + } + + template <typename T, typename U> + using TConverter = std::function<U(const T&)>; + + template <typename T, typename U> + U Implicit(const T& v) { + return v; + } + + ui16 Days(const TInstant& v) { + return v.Days(); + } + + ui32 Seconds(const TInstant& v) { + return v.Seconds(); + } + + ui64 MicroSeconds(const TInstant& v) { + return v.MicroSeconds(); + } + + std::pair<ui64, ui64> Int128ToPair(const NYql::NDecimal::TInt128& v) { + return NYql::NDecimal::MakePair(v); + } + + TStringBuf BinaryJsonToStringBuf(const TMaybe<NBinaryJson::TBinaryJson>& v) { + Y_VERIFY(v.Defined()); + return TStringBuf(v->Data(), v->Size()); + } + + TStringBuf DyNumberToStringBuf(const TMaybe<TString>& v) { + Y_VERIFY(v.Defined()); + return TStringBuf(*v); + } + + template <typename T, typename U = T> + struct TCellMaker { + static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, U> conv = &Implicit<T, U>) { + T t; + if (!TryParse<T>(v, t)) { + return false; + } + + auto& u = *pool.Allocate<U>(); + u = conv(t); + c = TCell(reinterpret_cast<const char*>(&u), sizeof(u)); + + return true; + } + }; + + template <typename T> + struct TCellMaker<T, TStringBuf> { + static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, TStringBuf> conv = &Implicit<T, TStringBuf>) { + T t; + if (!TryParse<T>(v, t)) { + return false; + } + + const auto u = pool.AppendString(conv(t)); + c = TCell(u.data(), u.size()); + + return true; + } + }; + +} // anonymous + +bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeId type, TMemoryPool& pool) { + if (value == "null") { + return true; + } + + switch (type) { + case NScheme::NTypeIds::Bool: + return TCellMaker<bool>::Make(cell, value, pool); + case NScheme::NTypeIds::Int8: + return TCellMaker<i8>::Make(cell, value, pool); + case NScheme::NTypeIds::Uint8: + return TCellMaker<ui8>::Make(cell, value, pool); + case NScheme::NTypeIds::Int16: + return TCellMaker<i16>::Make(cell, value, pool); + case NScheme::NTypeIds::Uint16: + return TCellMaker<ui16>::Make(cell, value, pool); + case NScheme::NTypeIds::Int32: + return TCellMaker<i32>::Make(cell, value, pool); + case NScheme::NTypeIds::Uint32: + return TCellMaker<ui32>::Make(cell, value, pool); + case NScheme::NTypeIds::Int64: + return TCellMaker<i64>::Make(cell, value, pool); + case NScheme::NTypeIds::Uint64: + return TCellMaker<ui64>::Make(cell, value, pool); + case NScheme::NTypeIds::Float: + return TCellMaker<float>::Make(cell, value, pool); + case NScheme::NTypeIds::Double: + return TCellMaker<double>::Make(cell, value, pool); + case NScheme::NTypeIds::Date: + return TCellMaker<TInstant, ui16>::Make(cell, value, pool, &Days); + case NScheme::NTypeIds::Datetime: + return TCellMaker<TInstant, ui32>::Make(cell, value, pool, &Seconds); + case NScheme::NTypeIds::Timestamp: + return TCellMaker<TInstant, ui64>::Make(cell, value, pool, &MicroSeconds); + case NScheme::NTypeIds::Interval: + return TCellMaker<i64>::Make(cell, value, pool); + case NScheme::NTypeIds::String: + case NScheme::NTypeIds::String4k: + case NScheme::NTypeIds::String2m: + case NScheme::NTypeIds::Utf8: + case NScheme::NTypeIds::Yson: + case NScheme::NTypeIds::Json: + return TCellMaker<TString, TStringBuf>::Make(cell, value, pool); + case NScheme::NTypeIds::JsonDocument: + return TCellMaker<TMaybe<NBinaryJson::TBinaryJson>, TStringBuf>::Make(cell, value, pool, &BinaryJsonToStringBuf); + case NScheme::NTypeIds::DyNumber: + return TCellMaker<TMaybe<TString>, TStringBuf>::Make(cell, value, pool, &DyNumberToStringBuf); + case NScheme::NTypeIds::Decimal: + return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, &Int128ToPair); + default: + return false; + } +} + +bool CheckCellValue(const TCell& cell, NScheme::TTypeId type) { + if (cell.IsNull()) { + return true; + } + + switch (type) { + case NScheme::NTypeIds::Bool: + case NScheme::NTypeIds::Int8: + case NScheme::NTypeIds::Uint8: + case NScheme::NTypeIds::Int16: + case NScheme::NTypeIds::Uint16: + case NScheme::NTypeIds::Int32: + case NScheme::NTypeIds::Uint32: + case NScheme::NTypeIds::Int64: + case NScheme::NTypeIds::Uint64: + case NScheme::NTypeIds::Float: + case NScheme::NTypeIds::Double: + case NScheme::NTypeIds::String: + case NScheme::NTypeIds::String4k: + case NScheme::NTypeIds::String2m: + case NScheme::NTypeIds::JsonDocument: // checked at parsing time + case NScheme::NTypeIds::DyNumber: // checked at parsing time + return true; + case NScheme::NTypeIds::Date: + return cell.AsValue<ui16>() < NUdf::MAX_DATE; + case NScheme::NTypeIds::Datetime: + return cell.AsValue<ui32>() < NUdf::MAX_DATETIME; + case NScheme::NTypeIds::Timestamp: + return cell.AsValue<ui64>() < NUdf::MAX_TIMESTAMP; + case NScheme::NTypeIds::Interval: + return (ui64)std::abs(cell.AsValue<i64>()) < NUdf::MAX_TIMESTAMP; + case NScheme::NTypeIds::Utf8: + return NYql::IsUtf8(cell.AsBuf()); + case NScheme::NTypeIds::Yson: + return NYql::NDom::IsValidYson(cell.AsBuf()); + case NScheme::NTypeIds::Json: + return NYql::NDom::IsValidJson(cell.AsBuf()); + case NScheme::NTypeIds::Decimal: + return !NYql::NDecimal::IsError(cell.AsValue<NYql::NDecimal::TInt128>()); + default: + return false; + } +} + bool TYdbDump::ParseLine(TStringBuf line, const std::vector<std::pair<i32, ui32>>& columnOrderTypes, TMemoryPool& pool, TVector<TCell>& keys, TVector<TCell>& values, TString& strError, ui64& numBytes) { diff --git a/ydb/core/io_formats/ya.make b/ydb/core/io_formats/ya.make index 62f53801f80..10e21d69d04 100644 --- a/ydb/core/io_formats/ya.make +++ b/ydb/core/io_formats/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -SRCS( +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +SRCS( csv_ydb_dump.cpp csv_arrow.cpp -) - -PEERDIR( - contrib/libs/double-conversion - library/cpp/string_utils/quote +) + +PEERDIR( + contrib/libs/double-conversion + library/cpp/string_utils/quote ydb/core/formats ydb/core/scheme ydb/library/binary_json @@ -22,8 +22,8 @@ PEERDIR( ydb/library/yql/public/udf ydb/library/yql/utils ydb/public/lib/scheme_types -) - +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/kesus/proxy/proxy.cpp b/ydb/core/kesus/proxy/proxy.cpp index 45b79611184..9633816e60a 100644 --- a/ydb/core/kesus/proxy/proxy.cpp +++ b/ydb/core/kesus/proxy/proxy.cpp @@ -174,15 +174,15 @@ private: case TSchemeCacheNavigate::EStatus::PathNotPath: LOG_TRACE_S(ctx, NKikimrServices::KESUS_PROXY, "Resolve did not find path " << msg->KesusPath.Quote() - << ": " << result.Status); + << ": " << result.Status); entry.LastError.SetStatus(Ydb::StatusIds::NOT_FOUND); entry.LastError.AddIssues()->set_message("Kesus not found"); break; default: LOG_ERROR_S(ctx, NKikimrServices::KESUS_PROXY, - "Kesus resolve failed: " << result.Status); + "Kesus resolve failed: " << result.Status); entry.LastError.SetStatus(Ydb::StatusIds::INTERNAL_ERROR); - entry.LastError.AddIssues()->set_message(ToString(result.Status)); + entry.LastError.AddIssues()->set_message(ToString(result.Status)); break; } if (entry.ProxyActor && entry.LastError.GetStatus() != Ydb::StatusIds::SUCCESS) { diff --git a/ydb/core/kqp/executer/kqp_table_resolver.cpp b/ydb/core/kqp/executer/kqp_table_resolver.cpp index 015feb50028..6d02c3d07c4 100644 --- a/ydb/core/kqp/executer/kqp_table_resolver.cpp +++ b/ydb/core/kqp/executer/kqp_table_resolver.cpp @@ -241,7 +241,7 @@ private: entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; entry.TableId = tableId; entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; request->ResultSet.emplace_back(std::move(entry)); } diff --git a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp index e240734caea..3148a2ec59b 100644 --- a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp @@ -57,8 +57,8 @@ TKikimrParamsMap GetParamsMap(const NYdb::TParams& params) { protoParam.mutable_value()->CopyFrom(NYdb::TProtoAccessor::GetProto(pair.second)); NKikimrMiniKQL::TParams mkqlParam; - ConvertYdbTypeToMiniKQLType(protoParam.type(), *mkqlParam.MutableType()); - ConvertYdbValueToMiniKQLValue(protoParam.type(), protoParam.value(), *mkqlParam.MutableValue()); + ConvertYdbTypeToMiniKQLType(protoParam.type(), *mkqlParam.MutableType()); + ConvertYdbValueToMiniKQLValue(protoParam.type(), protoParam.value(), *mkqlParam.MutableValue()); paramsMap.insert(std::make_pair(pair.first, mkqlParam)); } diff --git a/ydb/core/kqp/kqp_compile_request.cpp b/ydb/core/kqp/kqp_compile_request.cpp index 372b3fdad64..2c18af159bc 100644 --- a/ydb/core/kqp/kqp_compile_request.cpp +++ b/ydb/core/kqp/kqp_compile_request.cpp @@ -184,7 +184,7 @@ private: entry.RequestType = TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; entry.Operation = TSchemeCacheNavigate::EOp::OpTable; entry.SyncVersion = false; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; navigate->ResultSet.emplace_back(entry); } diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index c5032506426..47489a9bd54 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -1041,7 +1041,7 @@ public: } auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(Database); + ev->Record.SetDatabaseName(Database); if (UserToken) { ev->Record.SetUserToken(UserToken->Serialized); } @@ -1100,7 +1100,7 @@ public: std::pair<TString, TString> pathPair; if (createDir) { TString error; - if (!TrySplitPathByDb(metadata->Name, Database, pathPair, error)) { + if (!TrySplitPathByDb(metadata->Name, Database, pathPair, error)) { return MakeFuture(ResultFromError<TGenericResult>(error)); } } else { @@ -1141,7 +1141,7 @@ public: profiles.Load(configResult.Config->GetTableProfilesConfig()); auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(Database); + ev->Record.SetDatabaseName(Database); if (UserToken) { ev->Record.SetUserToken(UserToken->Serialized); } @@ -1154,15 +1154,15 @@ public: for (const auto& index : metadata->Indexes) { auto indexDesc = schemeTx.MutableCreateIndexedTable()->AddIndexDescription(); indexDesc->SetName(index.Name); - switch (index.Type) { - case NYql::TIndexDescription::EType::GlobalSync: + switch (index.Type) { + case NYql::TIndexDescription::EType::GlobalSync: indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobal); - break; - case NYql::TIndexDescription::EType::GlobalAsync: + break; + case NYql::TIndexDescription::EType::GlobalAsync: indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync); - break; - } - + break; + } + indexDesc->SetState(static_cast<::NKikimrSchemeOp::EIndexState>(index.State)); for (const auto& col : index.KeyColumns) { indexDesc->AddKeyColumnNames(col); @@ -2046,7 +2046,7 @@ private: } auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(Database); + ev->Record.SetDatabaseName(Database); if (UserToken) { ev->Record.SetUserToken(UserToken->Serialized); } @@ -2595,16 +2595,16 @@ private: } } - if (const auto& ttl = metadata->TableSettings.TtlSettings) { - if (ttl.IsSet()) { - ConvertTtlSettingsToProto(ttl.GetValueSet(), *proto.mutable_ttl_settings()); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = "Can't reset TTL settings"; - return false; - } - } - + if (const auto& ttl = metadata->TableSettings.TtlSettings) { + if (ttl.IsSet()) { + ConvertTtlSettingsToProto(ttl.GetValueSet(), *proto.mutable_ttl_settings()); + } else { + code = Ydb::StatusIds::BAD_REQUEST; + error = "Can't reset TTL settings"; + return false; + } + } + return true; } @@ -2618,7 +2618,7 @@ private: return false; } - TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); + TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); for (const auto& familySettings : createTableProto.column_families()) { if (!families.ApplyFamilySettings(familySettings, &code, &error)) { @@ -2630,7 +2630,7 @@ private: return false; } - if (!NGRpcService::FillCreateTableSettingsDesc(tableDesc, createTableProto, profiles, code, error, warnings)) { + if (!NGRpcService::FillCreateTableSettingsDesc(tableDesc, createTableProto, profiles, code, error, warnings)) { return false; } return true; diff --git a/ydb/core/kqp/kqp_metadata_loader.cpp b/ydb/core/kqp/kqp_metadata_loader.cpp index e56b072fb41..167b8df3c42 100644 --- a/ydb/core/kqp/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/kqp_metadata_loader.cpp @@ -2,7 +2,7 @@ #include "kqp_ic_gateway_actors.h" #include <ydb/core/base/path.h> - + #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> @@ -25,7 +25,7 @@ std::pair<TNavigate::TEntry, TString> CreateNavigateEntry(const TString& path, c entry.Path = SplitPath(path); entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpTable; entry.SyncVersion = true; - entry.ShowPrivatePath = settings.WithPrivateTables_; + entry.ShowPrivatePath = settings.WithPrivateTables_; return {entry, path}; } @@ -39,7 +39,7 @@ std::pair<TNavigate::TEntry, TString> CreateNavigateEntry(const std::pair<TIndex entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpList; entry.SyncVersion = true; - entry.ShowPrivatePath = settings.WithPrivateTables_; + entry.ShowPrivatePath = settings.WithPrivateTables_; return {entry, pair.second}; } @@ -87,7 +87,7 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache using EStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; using EKind = NSchemeCache::TSchemeCacheNavigate::EKind; - auto message = ToString(entry.Status); + auto message = ToString(entry.Status); switch (entry.Status) { case EStatus::Ok: @@ -106,7 +106,7 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache case EStatus::RedirectLookupError: return ResultFromError<TResult>(YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, message)); default: - return ResultFromError<TResult>(ToString(entry.Status)); + return ResultFromError<TResult>(ToString(entry.Status)); } YQL_ENSURE(entry.Kind == EKind::KindTable || entry.Kind == EKind::KindOlapTable); diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 70e41c959cd..035d6b77012 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -555,7 +555,7 @@ public: .PrimaryKey(settings.PrimaryKey.Cast()) .Settings(settings.Other) .Indexes(settings.Indexes.Cast()) - .Changefeeds(settings.Changefeeds.Cast()) + .Changefeeds(settings.Changefeeds.Cast()) .PartitionBy(settings.PartitionBy.Cast()) .ColumnFamilies(settings.ColumnFamilies.Cast()) .TableSettings(settings.TableSettings.Cast()) diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index b90124bd45f..ad360fcaebe 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -739,19 +739,19 @@ public: errText)); return SyncError(); } - } else if (name == "setTtlSettings") { - TTtlSettings ttlSettings; - TString error; - - YQL_ENSURE(setting.Value().Maybe<TCoNameValueTupleList>()); - if (!TTtlSettings::TryParse(setting.Value().Cast<TCoNameValueTupleList>(), ttlSettings, error)) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Invalid TTL settings: " << error)); - return SyncError(); - } - + } else if (name == "setTtlSettings") { + TTtlSettings ttlSettings; + TString error; + + YQL_ENSURE(setting.Value().Maybe<TCoNameValueTupleList>()); + if (!TTtlSettings::TryParse(setting.Value().Cast<TCoNameValueTupleList>(), ttlSettings, error)) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Invalid TTL settings: " << error)); + return SyncError(); + } + ConvertTtlSettingsToProto(ttlSettings, *alterTableRequest.mutable_set_ttl_settings()); - } else if (name == "resetTtlSettings") { + } else if (name == "resetTtlSettings") { alterTableRequest.mutable_drop_ttl_settings(); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), @@ -771,10 +771,10 @@ public: add_index->set_name(TString(columnTuple.Item(1).Cast<TCoAtom>().Value())); } else if (name == "indexType") { const auto type = TString(columnTuple.Item(1).Cast<TCoAtom>().Value()); - if (type == "syncGlobal") { + if (type == "syncGlobal") { add_index->mutable_global_index(); - } else if (type == "asyncGlobal") { - add_index->mutable_global_async_index(); + } else if (type == "asyncGlobal") { + add_index->mutable_global_async_index(); } else { ctx.AddError(TIssue(ctx.GetPosition(columnTuple.Item(1).Cast<TCoAtom>().Pos()), TStringBuilder() << "Unknown index type: " << type)); @@ -798,16 +798,16 @@ public: return SyncError(); } } - switch (add_index->type_case()) { - case Ydb::Table::TableIndex::kGlobalIndex: + switch (add_index->type_case()) { + case Ydb::Table::TableIndex::kGlobalIndex: *add_index->mutable_global_index() = Ydb::Table::GlobalIndex(); - break; - case Ydb::Table::TableIndex::kGlobalAsyncIndex: + break; + case Ydb::Table::TableIndex::kGlobalAsyncIndex: *add_index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; - default: - YQL_ENSURE(false, "Unknown index type: " << (ui32)add_index->type_case()); - } + break; + default: + YQL_ENSURE(false, "Unknown index type: " << (ui32)add_index->type_case()); + } } else if (name == "dropIndex") { auto nameNode = action.Value().Cast<TCoAtom>(); alterTableRequest.add_drop_indexes(TString(nameNode.Value())); diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index 1a8d3080fe8..c54d82a2cac 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -116,8 +116,8 @@ {"Index": 6, "Name": "Indexes", "Type": "TCoIndexList"}, {"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"}, {"Index": 8, "Name": "ColumnFamilies", "Type": "TExprList"}, - {"Index": 9, "Name": "TableSettings", "Type": "TCoNameValueTupleList"}, - {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"} + {"Index": 9, "Name": "TableSettings", "Type": "TCoNameValueTupleList"}, + {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"} ] }, { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index 38da23e5932..e49f84b5e1e 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -206,36 +206,36 @@ void IKikimrGateway::BuildIndexMetadata(TTableMetadataResult& loadTableMetadataR } } -bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error) { - using namespace NNodes; - - for (const auto& field : node) { - auto name = field.Name().Value(); - if (name == "columnName") { - YQL_ENSURE(field.Value().Maybe<TCoAtom>()); - settings.ColumnName = field.Value().Cast<TCoAtom>().StringValue(); - } else if (name == "expireAfter") { - YQL_ENSURE(field.Value().Maybe<TCoInterval>()); - auto value = FromString<i64>(field.Value().Cast<TCoInterval>().Literal().Value()); - if (value < 0) { - error = "Interval value cannot be negative"; - return false; - } - - settings.ExpireAfter = TDuration::FromValue(value); - } else { - error = TStringBuilder() << "Unknown field: " << name; - return false; - } - } - - return true; -} - +bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error) { + using namespace NNodes; + + for (const auto& field : node) { + auto name = field.Name().Value(); + if (name == "columnName") { + YQL_ENSURE(field.Value().Maybe<TCoAtom>()); + settings.ColumnName = field.Value().Cast<TCoAtom>().StringValue(); + } else if (name == "expireAfter") { + YQL_ENSURE(field.Value().Maybe<TCoInterval>()); + auto value = FromString<i64>(field.Value().Cast<TCoInterval>().Literal().Value()); + if (value < 0) { + error = "Interval value cannot be negative"; + return false; + } + + settings.ExpireAfter = TDuration::FromValue(value); + } else { + error = TStringBuilder() << "Unknown field: " << name; + return false; + } + } + + return true; +} + bool TTableSettings::IsSet() const { return CompactionPolicy || PartitionBy || AutoPartitioningBySize || UniformPartitions || PartitionAtKeys || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || KeyBloomFilter - || ReadReplicasSettings || TtlSettings; + || ReadReplicasSettings || TtlSettings; } EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 0dd41360792..7598494a2b9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -16,7 +16,7 @@ #include <library/cpp/threading/future/future.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/flags.h> #include <util/generic/map.h> #include <util/generic/maybe.h> @@ -136,15 +136,15 @@ struct TIndexDescription { DataColumns == other.DataColumns && Type == other.Type; } - - bool ItUsedForWrite() const { - switch (Type) { - case EType::GlobalSync: - return true; - case EType::GlobalAsync: - return false; - } - } + + bool ItUsedForWrite() const { + switch (Type) { + case EType::GlobalSync: + return true; + case EType::GlobalAsync: + return false; + } + } }; struct TColumnFamily { @@ -153,13 +153,13 @@ struct TColumnFamily { TMaybe<TString> Compression; }; -struct TTtlSettings { - TString ColumnName; - TDuration ExpireAfter; - - static bool TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error); -}; - +struct TTtlSettings { + TString ColumnName; + TDuration ExpireAfter; + + static bool TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error); +}; + struct TTableSettings { TMaybe<TString> CompactionPolicy; TVector<TString> PartitionBy; @@ -172,7 +172,7 @@ struct TTableSettings { TVector<TVector<std::pair<EDataSlot, TString>>> PartitionAtKeys; TMaybe<TString> KeyBloomFilter; TMaybe<TString> ReadReplicasSettings; - TResetableSetting<TTtlSettings, void> TtlSettings; + TResetableSetting<TTtlSettings, void> TtlSettings; bool IsSet() const; }; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index 5aa60832129..791a617c083 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -4,8 +4,8 @@ #include <ydb/core/kqp/kqp_impl.h> #include <ydb/core/kqp/kqp_metadata_loader.h> -#include <util/generic/maybe.h> - +#include <util/generic/maybe.h> + namespace NYql { using namespace NKikimr; @@ -218,7 +218,7 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient TString("Column2Index"), TVector<TString>{"Column2"}, dataColumns, - TIndexDescription::EType::GlobalSync, + TIndexDescription::EType::GlobalSync, TIndexDescription::EIndexState::Ready, 0, 0, @@ -244,7 +244,7 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient responseFuture.Wait(); auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); - if ((!shouldCreate && !createFolders) || (shouldCreate && !*shouldCreate)) { + if ((!shouldCreate && !createFolders) || (shouldCreate && !*shouldCreate)) { UNIT_ASSERT(!response.Success()); UNIT_ASSERT(HasIssue(response.Issues(), TIssuesIds::KIKIMR_SCHEME_ERROR)); } else { diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 2ba3ed20581..692165204ae 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -613,17 +613,17 @@ private: } for (const auto& index : create.Indexes()) { - const auto type = index.Type().Value(); - TIndexDescription::EType indexType; - - if (type == "syncGlobal") { - indexType = TIndexDescription::EType::GlobalSync; - } else if (type == "asyncGlobal") { - indexType = TIndexDescription::EType::GlobalAsync; - } else { - YQL_ENSURE(false, "Unknown index type: " << type); - } - + const auto type = index.Type().Value(); + TIndexDescription::EType indexType; + + if (type == "syncGlobal") { + indexType = TIndexDescription::EType::GlobalSync; + } else if (type == "asyncGlobal") { + indexType = TIndexDescription::EType::GlobalAsync; + } else { + YQL_ENSURE(false, "Unknown index type: " << type); + } + TVector<TString> indexColums; TVector<TString> dataColums; @@ -793,22 +793,22 @@ private: meta->TableSettings.ReadReplicasSettings = TString( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() ); - } else if (name == "setTtlSettings") { - TTtlSettings ttlSettings; - TString error; - - YQL_ENSURE(setting.Value().Maybe<TCoNameValueTupleList>()); - if (!TTtlSettings::TryParse(setting.Value().Cast<TCoNameValueTupleList>(), ttlSettings, error)) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Invalid TTL settings: " << error)); - return TStatus::Error; - } - - meta->TableSettings.TtlSettings.Set(ttlSettings); - } else if (name == "resetTtlSettings") { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't reset TTL settings")); - return TStatus::Error; + } else if (name == "setTtlSettings") { + TTtlSettings ttlSettings; + TString error; + + YQL_ENSURE(setting.Value().Maybe<TCoNameValueTupleList>()); + if (!TTtlSettings::TryParse(setting.Value().Cast<TCoNameValueTupleList>(), ttlSettings, error)) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Invalid TTL settings: " << error)); + return TStatus::Error; + } + + meta->TableSettings.TtlSettings.Set(ttlSettings); + } else if (name == "resetTtlSettings") { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't reset TTL settings")); + return TStatus::Error; } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), TStringBuilder() << "Unknown table profile setting: " << name)); diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp index 4305ac6c6c8..decd2cb3011 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp @@ -68,7 +68,7 @@ std::optional<ui32> TryDecodeYdbSessionId(const TString& sessionId) { } try { - NOperationId::TOperationId opId(sessionId); + NOperationId::TOperationId opId(sessionId); ui32 nodeId; const auto& nodeIds = opId.GetValue("node_id"); if (nodeIds.size() != 1) { @@ -105,10 +105,10 @@ bool IsSqlQuery(const NKikimrKqp::EQueryType& queryType) { TString EncodeSessionId(ui32 nodeId, const TString& id) { Ydb::TOperationId opId; - opId.SetKind(NOperationId::TOperationId::SESSION_YQL); - NOperationId::AddOptionalValue(opId, "node_id", ToString(nodeId)); - NOperationId::AddOptionalValue(opId, "id", Base64Encode(id)); - return NOperationId::ProtoToString(opId); + opId.SetKind(NOperationId::TOperationId::SESSION_YQL); + NOperationId::AddOptionalValue(opId, "node_id", ToString(nodeId)); + NOperationId::AddOptionalValue(opId, "id", Base64Encode(id)); + return NOperationId::ProtoToString(opId); } @@ -1114,8 +1114,8 @@ private: return false; } - /* - * TODO: disabled due to SLYDB-42. Remove (or rework) it after KIKIMR-9650 & KIKIMR-9652 + /* + * TODO: disabled due to SLYDB-42. Remove (or rework) it after KIKIMR-9650 & KIKIMR-9652 if (!Tenants.contains(database)) { TString error = TStringBuilder() << "Unexpected node for database: " << database; @@ -1126,7 +1126,7 @@ private: result.Error = error; return false; } - */ + */ } if (ShutdownRequested) { diff --git a/ydb/core/kqp/runtime/kqp_transport.cpp b/ydb/core/kqp/runtime/kqp_transport.cpp index 4459c321fd8..8c1b3262d0d 100644 --- a/ydb/core/kqp/runtime/kqp_transport.cpp +++ b/ydb/core/kqp/runtime/kqp_transport.cpp @@ -266,13 +266,13 @@ Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet(const TVector<NDqProto::TData for (auto& member : dstRowType->GetStruct().GetMember()) { auto* column = resultSet.add_columns(); column->set_name(member.GetName()); - ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type()); + ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type()); } } else { for (auto& member : srcRowType.GetStruct().GetMember()) { auto* column = resultSet.add_columns(); column->set_name(member.GetName()); - ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type()); + ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type()); } } diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index f646001c44f..9bc7889f92f 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -101,7 +101,7 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { ServerSettings->SetDomainName(settings.DomainRoot); ServerSettings->SetKqpSettings(effectiveKqpSettings); ServerSettings->SetAppConfig(settings.AppConfig); - ServerSettings->SetFeatureFlags(settings.FeatureFlags); + ServerSettings->SetFeatureFlags(settings.FeatureFlags); ServerSettings->SetNodeCount(settings.NodeCount); ServerSettings->SetEnableKqpSpilling(enableSpilling); ServerSettings->SetEnableDataColumnForIndexTable(true); diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index e3a2f73449e..c1d64b65c3e 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -72,7 +72,7 @@ const TString KikimrDefaultUtDomainRoot = "Root"; TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings(); -struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> { +struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> { NKikimrConfig::TAppConfig AppConfig; TVector<NKikimrKqp::TKqpSetting> KqpSettings; TString AuthToken; @@ -133,7 +133,7 @@ public: } bool IsUsingSnapshotReads() const { - return Server->GetRuntime()->GetAppData().FeatureFlags.GetEnableMvccSnapshotReads(); + return Server->GetRuntime()->GetAppData().FeatureFlags.GetEnableMvccSnapshotReads(); } private: @@ -160,33 +160,33 @@ struct TCollectedStreamResult { TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it); TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it); -enum class EIndexTypeSql { - Global, - GlobalSync, - GlobalAsync, -}; - -inline constexpr TStringBuf IndexTypeSqlString(EIndexTypeSql type) { - switch (type) { - case EIndexTypeSql::Global: - return "GLOBAL"; - case EIndexTypeSql::GlobalSync: - return "GLOBAL SYNC"; - case EIndexTypeSql::GlobalAsync: - return "GLOBAL ASYNC"; - } -} - -inline NYdb::NTable::EIndexType IndexTypeSqlToIndexType(EIndexTypeSql type) { - switch (type) { - case EIndexTypeSql::Global: - case EIndexTypeSql::GlobalSync: - return NYdb::NTable::EIndexType::GlobalSync; - case EIndexTypeSql::GlobalAsync: - return NYdb::NTable::EIndexType::GlobalAsync; - } -} - +enum class EIndexTypeSql { + Global, + GlobalSync, + GlobalAsync, +}; + +inline constexpr TStringBuf IndexTypeSqlString(EIndexTypeSql type) { + switch (type) { + case EIndexTypeSql::Global: + return "GLOBAL"; + case EIndexTypeSql::GlobalSync: + return "GLOBAL SYNC"; + case EIndexTypeSql::GlobalAsync: + return "GLOBAL ASYNC"; + } +} + +inline NYdb::NTable::EIndexType IndexTypeSqlToIndexType(EIndexTypeSql type) { + switch (type) { + case EIndexTypeSql::Global: + case EIndexTypeSql::GlobalSync: + return NYdb::NTable::EIndexType::GlobalSync; + case EIndexTypeSql::GlobalAsync: + return NYdb::NTable::EIndexType::GlobalAsync; + } +} + TString ReformatYson(const TString& yson); void CompareYson(const TString& expected, const TString& actual); void CompareYson(const TString& expected, const NKikimrMiniKQL::TResult& actual); diff --git a/ydb/core/kqp/ut/kqp_acl_ut.cpp b/ydb/core/kqp/ut/kqp_acl_ut.cpp index dd0db8d792b..203c8ba6188 100644 --- a/ydb/core/kqp/ut/kqp_acl_ut.cpp +++ b/ydb/core/kqp/ut/kqp_acl_ut.cpp @@ -121,43 +121,43 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) { AssertSuccessResult(result); driver.Stop(true); } - - Y_UNIT_TEST(RecursiveCreateTableShouldSuccess) { - TKikimrRunner kikimr; - { - auto schemeClient = kikimr.GetSchemeClient(); - - AssertSuccessResult(schemeClient.MakeDirectory("/Root/PQ").ExtractValueSync()); - - NYdb::NScheme::TPermissions permissions("user0@builtin", {"ydb.deprecated.create_table"}); - AssertSuccessResult(schemeClient.ModifyPermissions("/Root/PQ", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync() - ); - } - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); + + Y_UNIT_TEST(RecursiveCreateTableShouldSuccess) { + TKikimrRunner kikimr; + { + auto schemeClient = kikimr.GetSchemeClient(); + + AssertSuccessResult(schemeClient.MakeDirectory("/Root/PQ").ExtractValueSync()); + + NYdb::NScheme::TPermissions permissions("user0@builtin", {"ydb.deprecated.create_table"}); + AssertSuccessResult(schemeClient.ModifyPermissions("/Root/PQ", + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync() + ); + } + + auto driverConfig = TDriverConfig() + .SetEndpoint(kikimr.GetEndpoint()) + .SetAuthToken("user0@builtin"); + auto driver = TDriver(driverConfig); auto db = NYdb::NTable::TTableClient(driver); - - auto session = db.CreateSession().GetValueSync().GetSession(); - - const char* queryTmpl = R"( + + auto session = db.CreateSession().GetValueSync().GetSession(); + + const char* queryTmpl = R"( PRAGMA kikimr.UseNewEngine = "true"; CREATE TABLE `/Root/PQ/%s` ( - id Int64, - name String, - primary key (id) - ); - )"; - - AssertSuccessResult(session.ExecuteSchemeQuery(Sprintf(queryTmpl, "table")).ExtractValueSync()); - AssertSuccessResult(session.ExecuteSchemeQuery(Sprintf(queryTmpl, "a/b/c/table")).ExtractValueSync()); - - driver.Stop(true); - } + id Int64, + name String, + primary key (id) + ); + )"; + + AssertSuccessResult(session.ExecuteSchemeQuery(Sprintf(queryTmpl, "table")).ExtractValueSync()); + AssertSuccessResult(session.ExecuteSchemeQuery(Sprintf(queryTmpl, "a/b/c/table")).ExtractValueSync()); + + driver.Stop(true); + } } } // namespace NKqp diff --git a/ydb/core/kqp/ut/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_ut.cpp index cacd84aa259..505cf09e6b8 100644 --- a/ydb/core/kqp/ut/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/kqp_indexes_ut.cpp @@ -13,8 +13,8 @@ #include <library/cpp/json/json_reader.h> -#include <util/string/printf.h> - +#include <util/string/printf.h> + namespace NKikimr { namespace NKqp { @@ -83,7 +83,7 @@ void CreateTableWithIndexWithState( indexName, {"value"}, TVector<TString>(), - NYql::TIndexDescription::EType::GlobalSync, + NYql::TIndexDescription::EType::GlobalSync, state, 0, 0, @@ -109,8 +109,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); @@ -176,8 +176,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); @@ -439,8 +439,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(DataColumnWriteNull, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -519,8 +519,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(DataColumnWrite, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -910,8 +910,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(DataColumnSelect, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1094,8 +1094,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SelectConcurentTX, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1175,8 +1175,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SelectConcurentTX2, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1256,8 +1256,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(UpsertWithoutExtraNullDelete, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1487,8 +1487,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_TWIN(KeyIndex, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1670,8 +1670,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_TWIN(KeyIndex2, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1807,8 +1807,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_TWIN(ReplaceWithoutExtraNullDelete, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -1855,7 +1855,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); UNIT_ASSERT(!stats.query_phases(2).table_access(1).has_deletes()); } @@ -1888,7 +1888,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).updates().rows(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).name(), "/Root/TestTable/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).updates().rows(), 1); UNIT_ASSERT(stats.query_phases(2).table_access(1).has_deletes()); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(1).deletes().rows(), 1); } @@ -1898,15 +1898,15 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexUpsert1DeleteUpdate, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - const auto& config = kikimr.GetTestServer().GetSettings().AppConfig; - auto& tableSettings = config.GetTableServiceConfig(); + const auto& config = kikimr.GetTestServer().GetSettings().AppConfig; + auto& tableSettings = config.GetTableServiceConfig(); bool useSchemeCacheMeta = tableSettings.GetUseSchemeCacheMetadata(); { @@ -1956,7 +1956,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { { auto result = session.DescribeTable("/Root/TestTable/Index/indexImplTable").ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); } { @@ -2086,8 +2086,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexUpsert2Update, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -2177,8 +2177,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexUpdateOnUsingIndex, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -2244,8 +2244,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexSelect, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -2446,8 +2446,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -2816,8 +2816,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexOrderBy2, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3009,8 +3009,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { Y_UNIT_TEST_QUAD(SecondaryIndexReplace, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3088,8 +3088,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(SecondaryIndexInsert1, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3134,8 +3134,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(MultipleSecondaryIndex, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3199,8 +3199,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(MultipleSecondaryIndexWithSameComulns, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3537,8 +3537,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(SecondaryIndexWithPrimaryKeySameComulns, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3811,8 +3811,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_TWIN(DeleteOnWithSubquery, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -3867,8 +3867,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4057,8 +4057,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(SecondaryIndexUsingInJoin2, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4194,8 +4194,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(ForbidViewModification, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4282,8 +4282,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_TWIN(ForbidDirectIndexTableCreation, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4326,15 +4326,15 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Value"}); auto result = session.CreateTable("/Root/TestTable/Index/indexImplTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); } } Y_UNIT_TEST_QUAD(DuplicateUpsert, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4370,8 +4370,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(DuplicateUpsertInterleave, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4408,8 +4408,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(DuplicateUpsertInterleaveParams, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4496,8 +4496,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST_QUAD(MultipleModifications, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4541,36 +4541,36 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda } } - template <bool WithMvcc, bool UseNewEngine> - void CreateTableWithIndexSQL(EIndexTypeSql type, bool enableAsyncIndexes = false) { - auto kqpSetting = NKikimrKqp::TKqpSetting(); - kqpSetting.SetName("_KqpYqlSyntaxVersion"); - kqpSetting.SetValue("1"); - + template <bool WithMvcc, bool UseNewEngine> + void CreateTableWithIndexSQL(EIndexTypeSql type, bool enableAsyncIndexes = false) { + auto kqpSetting = NKikimrKqp::TKqpSetting(); + kqpSetting.SetName("_KqpYqlSyntaxVersion"); + kqpSetting.SetValue("1"); + auto settings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) - .SetEnableAsyncIndexes(enableAsyncIndexes) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableAsyncIndexes(enableAsyncIndexes) .SetKqpSettings({kqpSetting}); - TKikimrRunner kikimr(settings); + TKikimrRunner kikimr(settings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - const auto typeStr = IndexTypeSqlString(type); - const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) - ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::GENERIC_ERROR) - : EStatus::SUCCESS; - - const TString createTableSql = Sprintf("CREATE TABLE `/Root/TestTable` (" + const auto typeStr = IndexTypeSqlString(type); + const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) + ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::GENERIC_ERROR) + : EStatus::SUCCESS; + + const TString createTableSql = Sprintf("CREATE TABLE `/Root/TestTable` (" " Key Int32, IndexA Int32, IndexB Int32, IndexC String, Value String," " PRIMARY KEY (Key)," - " INDEX SecondaryIndex1 %s ON (IndexA, IndexB)," - " INDEX SecondaryIndex2 %s ON (IndexC)" - ")", typeStr.data(), typeStr.data()); + " INDEX SecondaryIndex1 %s ON (IndexA, IndexB)," + " INDEX SecondaryIndex2 %s ON (IndexC)" + ")", typeStr.data(), typeStr.data()); { auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); } // Multiple create table requests with same scheme should be OK @@ -4578,20 +4578,20 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda auto result = session.ExecuteSchemeQuery(createTableSql).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), expectedStatus); + } + + if (type == EIndexTypeSql::GlobalAsync && !enableAsyncIndexes) { + return; } - - if (type == EIndexTypeSql::GlobalAsync && !enableAsyncIndexes) { - return; - } - + // Check we can use index (metadata is correct) - const TString query1(Q_(R"( + const TString query1(Q_(R"( UPSERT INTO `/Root/TestTable` (Key, IndexA, IndexB, IndexC, Value) VALUES (1, 11, 111, "a", "Value1"), (2, 22, 222, "b", "Value2"), (3, 33, 333, "c", "Value3"); - )")); + )")); auto result = session.ExecuteDataQuery( query1, @@ -4599,122 +4599,122 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - auto waitForAsyncIndexContent = [&session](const TString& indexImplTable, const TString& expected) { - while (true) { - const auto& yson = ReadTablePartToYson(session, indexImplTable); - if (yson == expected) { - break; - } - - Sleep(TDuration::Seconds(1)); - } - }; - - { - const TString indexImplTable = "/Root/TestTable/SecondaryIndex1/indexImplTable"; - const TString expected = R"([[[11];[111];[1]];[[22];[222];[2]];[[33];[333];[3]]])"; - if (!enableAsyncIndexes) { - UNIT_ASSERT_VALUES_EQUAL(ReadTablePartToYson(session, indexImplTable), expected); - } else { - waitForAsyncIndexContent(indexImplTable, expected); - } - } - { - const TString indexImplTable = "/Root/TestTable/SecondaryIndex2/indexImplTable"; - const TString expected = R"([[["a"];[1]];[["b"];[2]];[["c"];[3]]])"; - if (!enableAsyncIndexes) { - UNIT_ASSERT_VALUES_EQUAL(ReadTablePartToYson(session, indexImplTable), expected); - } else { - waitForAsyncIndexContent(indexImplTable, expected); - } + + auto waitForAsyncIndexContent = [&session](const TString& indexImplTable, const TString& expected) { + while (true) { + const auto& yson = ReadTablePartToYson(session, indexImplTable); + if (yson == expected) { + break; + } + + Sleep(TDuration::Seconds(1)); + } + }; + + { + const TString indexImplTable = "/Root/TestTable/SecondaryIndex1/indexImplTable"; + const TString expected = R"([[[11];[111];[1]];[[22];[222];[2]];[[33];[333];[3]]])"; + if (!enableAsyncIndexes) { + UNIT_ASSERT_VALUES_EQUAL(ReadTablePartToYson(session, indexImplTable), expected); + } else { + waitForAsyncIndexContent(indexImplTable, expected); + } } + { + const TString indexImplTable = "/Root/TestTable/SecondaryIndex2/indexImplTable"; + const TString expected = R"([[["a"];[1]];[["b"];[2]];[["c"];[3]]])"; + if (!enableAsyncIndexes) { + UNIT_ASSERT_VALUES_EQUAL(ReadTablePartToYson(session, indexImplTable), expected); + } else { + waitForAsyncIndexContent(indexImplTable, expected); + } + } + } + + Y_UNIT_TEST_QUAD(CreateTableWithImplicitSyncIndexSQL, WithMvcc, UseNewEngine) { + CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::Global); + } + + Y_UNIT_TEST_QUAD(CreateTableWithExplicitSyncIndexSQL, WithMvcc, UseNewEngine) { + CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalSync); + } + + Y_UNIT_TEST_QUAD(CreateTableWithAsyncIndexSQLShouldFail, WithMvcc, UseNewEngine) { + CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalAsync); } - Y_UNIT_TEST_QUAD(CreateTableWithImplicitSyncIndexSQL, WithMvcc, UseNewEngine) { - CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::Global); - } - - Y_UNIT_TEST_QUAD(CreateTableWithExplicitSyncIndexSQL, WithMvcc, UseNewEngine) { - CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalSync); - } - - Y_UNIT_TEST_QUAD(CreateTableWithAsyncIndexSQLShouldFail, WithMvcc, UseNewEngine) { - CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalAsync); - } - - Y_UNIT_TEST_QUAD(CreateTableWithAsyncIndexSQLShouldSucceed, WithMvcc, UseNewEngine) { - CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalAsync, true); - } - - template <bool WithMvcc, bool UseNewEngine> - void SelectFromAsyncIndexedTable() { - auto kqpSetting = NKikimrKqp::TKqpSetting(); - kqpSetting.SetName("_KqpYqlSyntaxVersion"); - kqpSetting.SetValue("1"); - - auto settings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) - .SetEnableAsyncIndexes(true) - .SetKqpSettings({kqpSetting}); - TKikimrRunner kikimr(settings); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - auto result = session.ExecuteSchemeQuery(R"( - CREATE TABLE `/Root/TestTable` ( - Key Int32, Index Int32, Value String, - PRIMARY KEY (Key), - INDEX SecondaryIndex GLOBAL ASYNC ON (Index) - ) - )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const auto query = Q_("SELECT * FROM `/Root/TestTable` VIEW SecondaryIndex WHERE Index == 1;"); - auto queryId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery(); - - const auto variants = TVector<std::pair<TTxSettings, EStatus>>{ + Y_UNIT_TEST_QUAD(CreateTableWithAsyncIndexSQLShouldSucceed, WithMvcc, UseNewEngine) { + CreateTableWithIndexSQL<WithMvcc, UseNewEngine>(EIndexTypeSql::GlobalAsync, true); + } + + template <bool WithMvcc, bool UseNewEngine> + void SelectFromAsyncIndexedTable() { + auto kqpSetting = NKikimrKqp::TKqpSetting(); + kqpSetting.SetName("_KqpYqlSyntaxVersion"); + kqpSetting.SetValue("1"); + + auto settings = TKikimrSettings() + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableAsyncIndexes(true) + .SetKqpSettings({kqpSetting}); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"( + CREATE TABLE `/Root/TestTable` ( + Key Int32, Index Int32, Value String, + PRIMARY KEY (Key), + INDEX SecondaryIndex GLOBAL ASYNC ON (Index) + ) + )").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + const auto query = Q_("SELECT * FROM `/Root/TestTable` VIEW SecondaryIndex WHERE Index == 1;"); + auto queryId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery(); + + const auto variants = TVector<std::pair<TTxSettings, EStatus>>{ {TTxSettings::SerializableRW(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR}, {TTxSettings::OnlineRO(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR}, {TTxSettings::StaleRO(), EStatus::SUCCESS}, - }; - - for (const auto& [settings, status] : variants) { - { - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(settings).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, TStringBuilder() << "Unexpected status #1" - << ": expected# " << status - << ": got# " << result.GetStatus() - << ", settings# " << settings - << ", issues# " << result.GetIssues().ToString()); - } - - { - auto result = queryId.Execute(TTxControl::BeginTx(settings).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, TStringBuilder() << "Unexpected status #2" - << ": expected# " << status - << ": got# " << result.GetStatus() - << ", settings# " << settings - << ", issues# " << result.GetIssues().ToString()); - } - } - } - - Y_UNIT_TEST_QUAD(SelectFromAsyncIndexedTable, WithMvcc, UseNewEngine) { - SelectFromAsyncIndexedTable<WithMvcc, UseNewEngine>(); - } - + }; + + for (const auto& [settings, status] : variants) { + { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(settings).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, TStringBuilder() << "Unexpected status #1" + << ": expected# " << status + << ": got# " << result.GetStatus() + << ", settings# " << settings + << ", issues# " << result.GetIssues().ToString()); + } + + { + auto result = queryId.Execute(TTxControl::BeginTx(settings).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, TStringBuilder() << "Unexpected status #2" + << ": expected# " << status + << ": got# " << result.GetStatus() + << ", settings# " << settings + << ", issues# " << result.GetIssues().ToString()); + } + } + } + + Y_UNIT_TEST_QUAD(SelectFromAsyncIndexedTable, WithMvcc, UseNewEngine) { + SelectFromAsyncIndexedTable<WithMvcc, UseNewEngine>(); + } + Y_UNIT_TEST_QUAD(InnerJoinWithNonIndexWherePredicate, WithMvcc, UseNewEngine) { auto setting = NKikimrKqp::TKqpSetting(); setting.SetName("_KqpYqlSyntaxVersion"); setting.SetValue("1"); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -4799,8 +4799,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda setting.SetName("_KqpYqlSyntaxVersion"); setting.SetValue("1"); auto serverSettings = TKikimrSettings() - .SetEnableMvcc(WithMvcc) - .SetEnableMvccSnapshotReads(WithMvcc) + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_mvcc_ut.cpp b/ydb/core/kqp/ut/kqp_mvcc_ut.cpp index 54ba5df3945..571b39df3aa 100644 --- a/ydb/core/kqp/ut/kqp_mvcc_ut.cpp +++ b/ydb/core/kqp/ut/kqp_mvcc_ut.cpp @@ -11,8 +11,8 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpSnapshotRead) { Y_UNIT_TEST_NEW_ENGINE(TestSnapshotExpiration) { TKikimrRunner kikimr(TKikimrSettings() - .SetEnableMvcc(true) - .SetEnableMvccSnapshotReads(true) + .SetEnableMvcc(true) + .SetEnableMvccSnapshotReads(true) .SetKeepSnapshotTimeout(TDuration::Seconds(1))); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); @@ -62,7 +62,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { } Y_UNIT_TEST_NEW_ENGINE(ReadOnlyTxCommitsOnConcurrentWrite) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); + TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); @@ -120,7 +120,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { } Y_UNIT_TEST_NEW_ENGINE(ReadOnlyTxWithIndexCommitsOnConcurrentWrite) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); + TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); @@ -181,7 +181,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { } Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite1) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); + TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); @@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { } Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite2) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); + TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); @@ -256,7 +256,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) { } Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite3) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); + TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true)); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG); // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG); diff --git a/ydb/core/kqp/ut/kqp_olap_ut.cpp b/ydb/core/kqp/ut/kqp_olap_ut.cpp index 738a71dee83..908fc34a13d 100644 --- a/ydb/core/kqp/ut/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/kqp_olap_ut.cpp @@ -400,10 +400,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(SimpleQueryOlap) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -432,10 +432,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(SimpleLookupOlap) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -463,10 +463,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(SimpleRangeOlap) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -495,10 +495,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(CompositeRangeOlap) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -654,10 +654,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(QueryOltpAndOlap) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -687,10 +687,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(EmptyRange) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); WriteTestData(kikimr, "/Root/olapStore/olapTable", 10000, 3000000, 1000); @@ -710,10 +710,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(Aggregation) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); // EnableDebugLogging(kikimr); @@ -787,9 +787,9 @@ Y_UNIT_TEST_SUITE(KqpOlap) { static bool enableLog = false; auto doTest = [](std::optional<bool> viaSettings, std::optional<bool> viaPragma, bool pushdownPresent) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); if (enableLog) { Cerr << "Run test:" << Endl; @@ -810,10 +810,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { auto setting = NKikimrKqp::TKqpSetting(); setting.SetName("_KqpPushOlapProcess"); setting.SetValue(viaSettings.value() ? "true" : "false"); - settings.KqpSettings = { setting }; + settings.KqpSettings = { setting }; } - TKikimrRunner kikimr(settings); + TKikimrRunner kikimr(settings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); auto client = kikimr.GetTableClient(); @@ -883,10 +883,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(PKDescScan) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); TStreamExecScanQuerySettings scanSettings; scanSettings.Explain(true); @@ -949,10 +949,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(ExtractRanges) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 2000); @@ -977,10 +977,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(ExtractRangesReverse) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 2000); @@ -1008,10 +1008,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(PredicatePushdown) { constexpr bool logQueries = false; - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); TStreamExecScanQuerySettings scanSettings; scanSettings.Explain(true); @@ -1130,10 +1130,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysView) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); for (ui64 i = 0; i < 100; ++i) { @@ -1171,10 +1171,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysViewTable) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr, "olapTable_1"); CreateTestOlapTable(kikimr, "olapTable_2"); @@ -1232,10 +1232,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysViewColumns) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); for (ui64 i = 0; i < 10; ++i) { @@ -1292,10 +1292,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysViewRanges) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr, "olapTable_1"); CreateTestOlapTable(kikimr, "olapTable_2"); @@ -1375,10 +1375,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysViewFilter) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr); for (ui64 i = 0; i < 10; ++i) { @@ -1449,10 +1449,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(StatsSysViewAggregation) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); CreateTestOlapTable(kikimr, "olapTable_1"); CreateTestOlapTable(kikimr, "olapTable_2"); @@ -1626,10 +1626,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(PredicatePushdownWithParameters) { constexpr bool logQueries = true; - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); TStreamExecScanQuerySettings scanSettings; scanSettings.Explain(true); @@ -1710,10 +1710,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(PredicatePushdownParameterTypesValidation) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); TStreamExecScanQuerySettings scanSettings; scanSettings.Explain(true); @@ -1767,10 +1767,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } Y_UNIT_TEST(PredicatePushdownCastErrors) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); TStreamExecScanQuerySettings scanSettings; scanSettings.Explain(true); diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index 810a42e1baa..347ca354412 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -1771,10 +1771,10 @@ Y_UNIT_TEST_SUITE(KqpScan) { } Y_UNIT_TEST(LMapFunction) { - auto settings = TKikimrSettings() - .SetWithSampleTables(false) - .SetEnableOlapSchemaOperations(true); - TKikimrRunner kikimr(settings); + auto settings = TKikimrSettings() + .SetWithSampleTables(false) + .SetEnableOlapSchemaOperations(true); + TKikimrRunner kikimr(settings); auto tableClient = kikimr.GetTableClient(); auto session = tableClient.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp index a548a62d36c..ca7ffa42a4f 100644 --- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp @@ -3,8 +3,8 @@ #include <library/cpp/threading/local_executor/local_executor.h> -#include <util/string/printf.h> - +#include <util/string/printf.h> + namespace NKikimr { namespace NKqp { @@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto& tableSettings = kikimr.GetTestServer().GetSettings().AppConfig.GetTableServiceConfig(); bool useSchemeCacheMeta = tableSettings.GetUseSchemeCacheMetadata(); - + auto result = session.ExecuteDataQuery(R"( SELECT * FROM [/Root/KeyValue]; )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); @@ -483,46 +483,46 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - void AlterTableSetttings( - NYdb::NTable::TSession& session, const TString& tableName, - const THashMap<TString, TString>& settings, bool compat, - EStatus expectedStatus = EStatus::SUCCESS, const TString& expectedMessage = {}) { - - auto query = TStringBuilder() - << "--!syntax_v1" << Endl - << "ALTER TABLE `" << tableName << "` "; - - if (compat) { - query << "SET ("; - } - - bool needComma = false; - for (const auto& [key, value] : settings) { - if (needComma) { - query << ","; - } - - needComma = true; - - if (compat) { - query << key << "=" << value; - } else { - query << " SET " << key << " " << value; - } - } - - if (compat) { - query << ")"; - } - - const auto result = session.ExecuteSchemeQuery(query << ";").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), expectedStatus, result.GetIssues().ToString()); - if (expectedMessage) { - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), expectedMessage, "Unexpected error message"); - } - } - - void CreateTableWithCompactionPolicy(bool compat) { + void AlterTableSetttings( + NYdb::NTable::TSession& session, const TString& tableName, + const THashMap<TString, TString>& settings, bool compat, + EStatus expectedStatus = EStatus::SUCCESS, const TString& expectedMessage = {}) { + + auto query = TStringBuilder() + << "--!syntax_v1" << Endl + << "ALTER TABLE `" << tableName << "` "; + + if (compat) { + query << "SET ("; + } + + bool needComma = false; + for (const auto& [key, value] : settings) { + if (needComma) { + query << ","; + } + + needComma = true; + + if (compat) { + query << key << "=" << value; + } else { + query << " SET " << key << " " << value; + } + } + + if (compat) { + query << ")"; + } + + const auto result = session.ExecuteSchemeQuery(query << ";").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), expectedStatus, result.GetIssues().ToString()); + if (expectedMessage) { + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), expectedMessage, "Unexpected error message"); + } + } + + void CreateTableWithCompactionPolicy(bool compat) { TKikimrRunner kikimr = TKikimrRunner(); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -545,8 +545,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 2); } - - AlterTableSetttings(session, tableName, {{"COMPACTION_POLICY", "\"default\""}}, compat); + + AlterTableSetttings(session, tableName, {{"COMPACTION_POLICY", "\"default\""}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() @@ -554,14 +554,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateTableWithCompactionPolicyUncompat) { - CreateTableWithCompactionPolicy(false); - } - - Y_UNIT_TEST(CreateTableWithCompactionPolicyCompat) { - CreateTableWithCompactionPolicy(true); - } - + Y_UNIT_TEST(CreateTableWithCompactionPolicyUncompat) { + CreateTableWithCompactionPolicy(false); + } + + Y_UNIT_TEST(CreateTableWithCompactionPolicyCompat) { + CreateTableWithCompactionPolicy(true); + } + Y_UNIT_TEST(CreateAndAlterTableWithPartitionBy) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -637,7 +637,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - void CreateAndAlterTableWithPartitioningBySize(bool compat) { + void CreateAndAlterTableWithPartitioningBySize(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -667,7 +667,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); { TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); @@ -679,15 +679,15 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateAndAlterTableWithPartitioningBySizeUncompat) { - CreateAndAlterTableWithPartitioningBySize(false); - } - - Y_UNIT_TEST(CreateAndAlterTableWithPartitioningBySizeCompat) { - CreateAndAlterTableWithPartitioningBySize(true); - } - - void CreateAndAlterTableWithPartitionSize(bool compat) { + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningBySizeUncompat) { + CreateAndAlterTableWithPartitioningBySize(false); + } + + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningBySizeCompat) { + CreateAndAlterTableWithPartitioningBySize(true); + } + + void CreateAndAlterTableWithPartitionSize(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -727,10 +727,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 1000 * 1024 * 1024); } - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "0"}}, compat, - EStatus::GENERIC_ERROR, "Can't set preferred partition size to 0"); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "0"}}, compat, + EStatus::GENERIC_ERROR, "Can't set preferred partition size to 0"); - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() @@ -738,7 +738,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 0); } - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "500"}}, compat); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "500"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() @@ -747,14 +747,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateAndAlterTableWithPartitionSizeUncompat) { - CreateAndAlterTableWithPartitionSize(false); - } - - Y_UNIT_TEST(CreateAndAlterTableWithPartitionSizeCompat) { - CreateAndAlterTableWithPartitionSize(true); - } - + Y_UNIT_TEST(CreateAndAlterTableWithPartitionSizeUncompat) { + CreateAndAlterTableWithPartitionSize(false); + } + + Y_UNIT_TEST(CreateAndAlterTableWithPartitionSizeCompat) { + CreateAndAlterTableWithPartitionSize(true); + } + Y_UNIT_TEST(RenameTable) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -866,7 +866,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } - void CreateAndAlterTableWithPartitioningByLoad(bool compat) { + void CreateAndAlterTableWithPartitioningByLoad(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -893,7 +893,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(enabled, true); } - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_LOAD", "DISABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_LOAD", "DISABLED"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); bool enabled = describeResult->Record.GetPathDescription().GetTable() @@ -901,7 +901,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(enabled, false); } - AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_LOAD", "ENABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_LOAD", "ENABLED"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); bool enabled = describeResult->Record.GetPathDescription().GetTable() @@ -910,15 +910,15 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateAndAlterTableWithPartitioningByLoadUncompat) { - CreateAndAlterTableWithPartitioningByLoad(false); - } - - Y_UNIT_TEST(CreateAndAlterTableWithPartitioningByLoadCompat) { - CreateAndAlterTableWithPartitioningByLoad(true); - } - - void CreateAndAlterTableWithMinMaxPartitions(bool compat) { + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningByLoadUncompat) { + CreateAndAlterTableWithPartitioningByLoad(false); + } + + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningByLoadCompat) { + CreateAndAlterTableWithPartitioningByLoad(true); + } + + void CreateAndAlterTableWithMinMaxPartitions(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -961,15 +961,15 @@ Y_UNIT_TEST_SUITE(KqpScheme) { .GetPartitioningPolicy().GetMaxPartitionsCount(), 100); } - AlterTableSetttings(session, tableName, { - {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, - {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "0"} - }, compat, EStatus::GENERIC_ERROR, "Can't set max partition count to 0"); + AlterTableSetttings(session, tableName, { + {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, + {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "0"} + }, compat, EStatus::GENERIC_ERROR, "Can't set max partition count to 0"); - AlterTableSetttings(session, tableName, { - {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, - {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "50"} - }, compat); + AlterTableSetttings(session, tableName, { + {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, + {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "50"} + }, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); @@ -980,15 +980,15 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateAndAlterTableWithMinMaxPartitionsUncompat) { - CreateAndAlterTableWithMinMaxPartitions(false); - } - - Y_UNIT_TEST(CreateAndAlterTableWithMinMaxPartitionsCompat) { - CreateAndAlterTableWithMinMaxPartitions(true); - } - - void CreateAndAlterTableWithBloomFilter(bool compat) { + Y_UNIT_TEST(CreateAndAlterTableWithMinMaxPartitionsUncompat) { + CreateAndAlterTableWithMinMaxPartitions(false); + } + + Y_UNIT_TEST(CreateAndAlterTableWithMinMaxPartitionsCompat) { + CreateAndAlterTableWithMinMaxPartitions(true); + } + + void CreateAndAlterTableWithBloomFilter(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1012,14 +1012,14 @@ Y_UNIT_TEST_SUITE(KqpScheme) { .GetEnableFilterByKey(), true); } - AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "DISABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "DISABLED"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() .GetEnableFilterByKey(), false); } - AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "ENABLED"}}, compat); + AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "ENABLED"}}, compat); { auto describeResult = kikimr.GetTestClient().Ls(tableName); UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() @@ -1027,15 +1027,15 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateAndAlterTableWithBloomFilterUncompat) { - CreateAndAlterTableWithBloomFilter(false); - } - - Y_UNIT_TEST(CreateAndAlterTableWithBloomFilterCompat) { - CreateAndAlterTableWithBloomFilter(true); - } - - void CreateTableWithReadReplicas(bool compat) { + Y_UNIT_TEST(CreateAndAlterTableWithBloomFilterUncompat) { + CreateAndAlterTableWithBloomFilter(false); + } + + Y_UNIT_TEST(CreateAndAlterTableWithBloomFilterCompat) { + CreateAndAlterTableWithBloomFilter(true); + } + + void CreateTableWithReadReplicas(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1097,7 +1097,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL(readReplicasSettings->GetReadReplicasCount(), 1); } - AlterTableSetttings(session, tableName, {{"READ_REPLICAS_SETTINGS", "\"PER_AZ:2\""}}, compat); + AlterTableSetttings(session, tableName, {{"READ_REPLICAS_SETTINGS", "\"PER_AZ:2\""}}, compat); { const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); const auto readReplicasSettings = tableDesc.GetReadReplicasSettings(); @@ -1107,153 +1107,153 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateTableWithReadReplicasUncompat) { - CreateTableWithReadReplicas(false); - } - - Y_UNIT_TEST(CreateTableWithReadReplicasCompat) { - CreateTableWithReadReplicas(true); - } - - void CreateTableWithTtlSettings(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithTtlSettings"; - - auto createTable = [&](const TString& ttlSettings, EStatus expectedStatus = EStatus::SUCCESS, const TString& expectedMessage = {}) { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Ts Timestamp, - StringValue String, - Uint32Value Uint32, - Uint64Value Uint64, - DyNumberValue DyNumber, - PRIMARY KEY (Key) - ) WITH ( - TTL = )" << ttlSettings << R"( - ))"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), expectedStatus, result.GetIssues().ToString()); - if (expectedMessage) { - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), expectedMessage, "Unexpected error message"); - } - }; - - createTable(R"(DateTime::IntervalFromDays(1) ON Ts)", EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); - - createTable(R"("P1D" ON Ts)", EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); - - createTable(R"(Interval("-P1D") ON Ts)", EStatus::GENERIC_ERROR, "Interval value cannot be negative"); - - createTable(R"(Interval("P1D") ON CreatedAt)", EStatus::GENERIC_ERROR, "Cannot enable TTL on unknown column"); - - createTable(R"(Interval("P1D") ON StringValue)", EStatus::GENERIC_ERROR, "Unsupported column type"); - - createTable(R"(Interval("P1D") ON Uint32Value)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - createTable(R"(Interval("P1D") ON Uint64Value)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - createTable(R"(Interval("P1D") ON DyNumberValue)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - - createTable(R"(Interval("P1D") ON Ts)"); - { - auto result = kikimr.GetTestClient().Ls(tableName); - const auto& table = result->Record.GetPathDescription().GetTable(); - UNIT_ASSERT(table.HasTTLSettings()); - - const auto& ttl = table.GetTTLSettings(); - UNIT_ASSERT(ttl.HasEnabled()); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "Ts"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 86'400); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` RESET (TTL);)"; - const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto result = kikimr.GetTestClient().Ls(tableName); - const auto& table = result->Record.GetPathDescription().GetTable(); - UNIT_ASSERT(table.HasTTLSettings()); - - const auto& ttl = table.GetTTLSettings(); - UNIT_ASSERT(ttl.HasDisabled()); - } - - AlterTableSetttings(session, tableName, {{"TTL", R"(DateTime::IntervalFromDays(1) ON Ts)"}}, compat, - EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); - - AlterTableSetttings(session, tableName, {{"TTL", R"("-P1D" ON Ts)"}}, compat, - EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); - - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("-P1D") ON Ts)"}}, compat, - EStatus::GENERIC_ERROR, "Interval value cannot be negative"); - - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON CreatedAt)"}}, compat, - EStatus::GENERIC_ERROR, "Cannot enable TTL on unknown column"); - - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON StringValue)"}}, compat, - EStatus::GENERIC_ERROR, "Unsupported column type"); - - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Uint32Value)"}}, compat, - EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Uint64Value)"}}, compat, - EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON DyNumberValue)"}}, compat, - EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); - - AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Ts)"}}, compat); - { - auto result = kikimr.GetTestClient().Ls(tableName); - const auto& table = result->Record.GetPathDescription().GetTable(); - UNIT_ASSERT(table.HasTTLSettings()); - - const auto& ttl = table.GetTTLSettings(); - UNIT_ASSERT(ttl.HasEnabled()); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "Ts"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 0); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` DROP COLUMN Ts;)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Can't drop TTL column"); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` RESET (TTL);)"; - const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` DROP COLUMN Ts;)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(CreateTableWithTtlSettingsUncompat) { - CreateTableWithTtlSettings(false); - } - - Y_UNIT_TEST(CreateTableWithTtlSettingsCompat) { - CreateTableWithTtlSettings(true); - } - - void CreateTableWithUniformPartitions(bool compat) { + Y_UNIT_TEST(CreateTableWithReadReplicasUncompat) { + CreateTableWithReadReplicas(false); + } + + Y_UNIT_TEST(CreateTableWithReadReplicasCompat) { + CreateTableWithReadReplicas(true); + } + + void CreateTableWithTtlSettings(bool compat) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithTtlSettings"; + + auto createTable = [&](const TString& ttlSettings, EStatus expectedStatus = EStatus::SUCCESS, const TString& expectedMessage = {}) { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Ts Timestamp, + StringValue String, + Uint32Value Uint32, + Uint64Value Uint64, + DyNumberValue DyNumber, + PRIMARY KEY (Key) + ) WITH ( + TTL = )" << ttlSettings << R"( + ))"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), expectedStatus, result.GetIssues().ToString()); + if (expectedMessage) { + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), expectedMessage, "Unexpected error message"); + } + }; + + createTable(R"(DateTime::IntervalFromDays(1) ON Ts)", EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); + + createTable(R"("P1D" ON Ts)", EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); + + createTable(R"(Interval("-P1D") ON Ts)", EStatus::GENERIC_ERROR, "Interval value cannot be negative"); + + createTable(R"(Interval("P1D") ON CreatedAt)", EStatus::GENERIC_ERROR, "Cannot enable TTL on unknown column"); + + createTable(R"(Interval("P1D") ON StringValue)", EStatus::GENERIC_ERROR, "Unsupported column type"); + + createTable(R"(Interval("P1D") ON Uint32Value)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + createTable(R"(Interval("P1D") ON Uint64Value)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + createTable(R"(Interval("P1D") ON DyNumberValue)", EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + + createTable(R"(Interval("P1D") ON Ts)"); + { + auto result = kikimr.GetTestClient().Ls(tableName); + const auto& table = result->Record.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasTTLSettings()); + + const auto& ttl = table.GetTTLSettings(); + UNIT_ASSERT(ttl.HasEnabled()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "Ts"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 86'400); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(` RESET (TTL);)"; + const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = kikimr.GetTestClient().Ls(tableName); + const auto& table = result->Record.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasTTLSettings()); + + const auto& ttl = table.GetTTLSettings(); + UNIT_ASSERT(ttl.HasDisabled()); + } + + AlterTableSetttings(session, tableName, {{"TTL", R"(DateTime::IntervalFromDays(1) ON Ts)"}}, compat, + EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); + + AlterTableSetttings(session, tableName, {{"TTL", R"("-P1D" ON Ts)"}}, compat, + EStatus::GENERIC_ERROR, "Literal of Interval type is expected for TTL"); + + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("-P1D") ON Ts)"}}, compat, + EStatus::GENERIC_ERROR, "Interval value cannot be negative"); + + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON CreatedAt)"}}, compat, + EStatus::GENERIC_ERROR, "Cannot enable TTL on unknown column"); + + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON StringValue)"}}, compat, + EStatus::GENERIC_ERROR, "Unsupported column type"); + + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Uint32Value)"}}, compat, + EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Uint64Value)"}}, compat, + EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON DyNumberValue)"}}, compat, + EStatus::GENERIC_ERROR, "'ValueSinceUnixEpochModeSettings' should be specified"); + + AlterTableSetttings(session, tableName, {{"TTL", R"(Interval("P0D") ON Ts)"}}, compat); + { + auto result = kikimr.GetTestClient().Ls(tableName); + const auto& table = result->Record.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasTTLSettings()); + + const auto& ttl = table.GetTTLSettings(); + UNIT_ASSERT(ttl.HasEnabled()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "Ts"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 0); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(` DROP COLUMN Ts;)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Can't drop TTL column"); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(` RESET (TTL);)"; + const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(` DROP COLUMN Ts;)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(CreateTableWithTtlSettingsUncompat) { + CreateTableWithTtlSettings(false); + } + + Y_UNIT_TEST(CreateTableWithTtlSettingsCompat) { + CreateTableWithTtlSettings(true); + } + + void CreateTableWithUniformPartitions(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1276,19 +1276,19 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); - AlterTableSetttings(session, tableName, {{"UNIFORM_PARTITIONS", "8"}}, compat, - EStatus::GENERIC_ERROR, "UNIFORM_PARTITIONS alter is not supported"); + AlterTableSetttings(session, tableName, {{"UNIFORM_PARTITIONS", "8"}}, compat, + EStatus::GENERIC_ERROR, "UNIFORM_PARTITIONS alter is not supported"); + } + + Y_UNIT_TEST(CreateTableWithUniformPartitionsUncompat) { + CreateTableWithUniformPartitions(false); + } + + Y_UNIT_TEST(CreateTableWithUniformPartitionsCompat) { + CreateTableWithUniformPartitions(true); } - Y_UNIT_TEST(CreateTableWithUniformPartitionsUncompat) { - CreateTableWithUniformPartitions(false); - } - - Y_UNIT_TEST(CreateTableWithUniformPartitionsCompat) { - CreateTableWithUniformPartitions(true); - } - - void CreateTableWithPartitionAtKeysSimple(bool compat) { + void CreateTableWithPartitionAtKeysSimple(bool compat) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1342,18 +1342,18 @@ Y_UNIT_TEST_SUITE(KqpScheme) { ++n; } - AlterTableSetttings(session, tableName, {{"PARTITION_AT_KEYS", "(100, 500, 1000, 10000)"}}, compat, - EStatus::GENERIC_ERROR, "PARTITION_AT_KEYS alter is not supported"); + AlterTableSetttings(session, tableName, {{"PARTITION_AT_KEYS", "(100, 500, 1000, 10000)"}}, compat, + EStatus::GENERIC_ERROR, "PARTITION_AT_KEYS alter is not supported"); + } + + Y_UNIT_TEST(CreateTableWithPartitionAtKeysSimpleUncompat) { + CreateTableWithPartitionAtKeysSimple(false); + } + + Y_UNIT_TEST(CreateTableWithPartitionAtKeysSimpleCompat) { + CreateTableWithPartitionAtKeysSimple(true); } - Y_UNIT_TEST(CreateTableWithPartitionAtKeysSimpleUncompat) { - CreateTableWithPartitionAtKeysSimple(false); - } - - Y_UNIT_TEST(CreateTableWithPartitionAtKeysSimpleCompat) { - CreateTableWithPartitionAtKeysSimple(true); - } - Y_UNIT_TEST(CreateTableWithPartitionAtKeysComplex) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -1740,38 +1740,38 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - void AlterTableAddIndex(EIndexTypeSql type, bool enableAsyncIndexes = false) { - TKikimrRunner kikimr(TKikimrSettings().SetEnableAsyncIndexes(enableAsyncIndexes)); + void AlterTableAddIndex(EIndexTypeSql type, bool enableAsyncIndexes = false) { + TKikimrRunner kikimr(TKikimrSettings().SetEnableAsyncIndexes(enableAsyncIndexes)); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); - - const auto typeStr = IndexTypeSqlString(type); - const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) - ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::UNSUPPORTED) - : EStatus::SUCCESS; + + const auto typeStr = IndexTypeSqlString(type); + const auto expectedStatus = (type == EIndexTypeSql::GlobalAsync) + ? (enableAsyncIndexes ? EStatus::SUCCESS : EStatus::UNSUPPORTED) + : EStatus::SUCCESS; { - auto status = session.ExecuteSchemeQuery(Sprintf(R"( + auto status = session.ExecuteSchemeQuery(Sprintf(R"( --!syntax_v1 ALTER TABLE `/Root/Test` ADD INDEX NameIndex %s ON (Name); - )", typeStr.data())).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), expectedStatus, status.GetIssues().ToString()); + )", typeStr.data())).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), expectedStatus, status.GetIssues().ToString()); } { TDescribeTableResult describe = session.DescribeTable("/Root/Test").GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); - - if (expectedStatus != EStatus::SUCCESS) { - UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 0); - return; - } - + + if (expectedStatus != EStatus::SUCCESS) { + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 0); + return; + } + UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 0); } @@ -1792,10 +1792,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } { - auto status = session.ExecuteSchemeQuery(Sprintf(R"( + auto status = session.ExecuteSchemeQuery(Sprintf(R"( --!syntax_v1 ALTER TABLE `/Root/Test` ADD INDEX NameIndex %s ON (Name) COVER (Amount); - )", typeStr.data())).ExtractValueSync(); + )", typeStr.data())).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString()); } @@ -1805,28 +1805,28 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto indexDesc = describe.GetTableDescription().GetIndexDescriptions(); UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "NameIndex"); - UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexType(), IndexTypeSqlToIndexType(type)); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1); UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 1); } } - Y_UNIT_TEST(AlterTableAddImplicitSyncIndex) { - AlterTableAddIndex(EIndexTypeSql::Global); - } - - Y_UNIT_TEST(AlterTableAddExplicitSyncIndex) { - AlterTableAddIndex(EIndexTypeSql::GlobalSync); - } - - Y_UNIT_TEST(AlterTableAddAsyncIndexShouldFail) { - AlterTableAddIndex(EIndexTypeSql::GlobalAsync); - } - - Y_UNIT_TEST(AlterTableAddAsyncIndexShouldSucceed) { - AlterTableAddIndex(EIndexTypeSql::GlobalAsync, true); - } - + Y_UNIT_TEST(AlterTableAddImplicitSyncIndex) { + AlterTableAddIndex(EIndexTypeSql::Global); + } + + Y_UNIT_TEST(AlterTableAddExplicitSyncIndex) { + AlterTableAddIndex(EIndexTypeSql::GlobalSync); + } + + Y_UNIT_TEST(AlterTableAddAsyncIndexShouldFail) { + AlterTableAddIndex(EIndexTypeSql::GlobalAsync); + } + + Y_UNIT_TEST(AlterTableAddAsyncIndexShouldSucceed) { + AlterTableAddIndex(EIndexTypeSql::GlobalAsync, true); + } + Y_UNIT_TEST(AlterTableWithDecimalColumn) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/metering/bill_record.cpp b/ydb/core/metering/bill_record.cpp index bb569996e89..04776d5ea00 100644 --- a/ydb/core/metering/bill_record.cpp +++ b/ydb/core/metering/bill_record.cpp @@ -1,62 +1,62 @@ #include "bill_record.h" - -#include <library/cpp/json/json_writer.h> - -#include <util/generic/size_literals.h> -#include <util/string/cast.h> - -namespace NKikimr { - -NJson::TJsonMap TBillRecord::TUsage::ToJson() const { - return NJson::TJsonMap{ - {"type", ::ToString(Type_)}, - {"unit", ::ToString(Unit_)}, - {"quantity", Quantity_}, - {"start", Start_.Seconds()}, - {"finish", Finish_.Seconds()}, - }; -} - -TString TBillRecord::TUsage::ToString() const { - return NJson::WriteJson(ToJson(), false); -} - -NJson::TJsonMap TBillRecord::ToJson() const { - return NJson::TJsonMap{ - {"version", Version_}, - {"id", Id_}, - {"schema", Schema_}, - {"cloud_id", CloudId_}, - {"folder_id", FolderId_}, - {"resource_id", ResourceId_}, - {"source_id", SourceId_}, - {"source_wt", SourceWt_.Seconds()}, - {"tags", Tags_}, - {"usage", Usage_.ToJson()}, - }; -} - -TString TBillRecord::ToString() const { - TStringStream out; - - auto json = ToJson(); - NJson::WriteJson(&out, &json, false, false, false); - - out << Endl; - return out.Str(); -} - -TBillRecord::TUsage TBillRecord::RequestUnits(ui64 quantity, TInstant start, TInstant finish) { - return TUsage() - .Type(TUsage::EType::Delta) - .Unit(TUsage::EUnit::RequestUnit) - .Quantity(quantity) - .Start(start) - .Finish(finish); -} - -TBillRecord::TUsage TBillRecord::RequestUnits(ui64 quantity, TInstant now) { - return RequestUnits(quantity, now, now); -} - + +#include <library/cpp/json/json_writer.h> + +#include <util/generic/size_literals.h> +#include <util/string/cast.h> + +namespace NKikimr { + +NJson::TJsonMap TBillRecord::TUsage::ToJson() const { + return NJson::TJsonMap{ + {"type", ::ToString(Type_)}, + {"unit", ::ToString(Unit_)}, + {"quantity", Quantity_}, + {"start", Start_.Seconds()}, + {"finish", Finish_.Seconds()}, + }; +} + +TString TBillRecord::TUsage::ToString() const { + return NJson::WriteJson(ToJson(), false); +} + +NJson::TJsonMap TBillRecord::ToJson() const { + return NJson::TJsonMap{ + {"version", Version_}, + {"id", Id_}, + {"schema", Schema_}, + {"cloud_id", CloudId_}, + {"folder_id", FolderId_}, + {"resource_id", ResourceId_}, + {"source_id", SourceId_}, + {"source_wt", SourceWt_.Seconds()}, + {"tags", Tags_}, + {"usage", Usage_.ToJson()}, + }; +} + +TString TBillRecord::ToString() const { + TStringStream out; + + auto json = ToJson(); + NJson::WriteJson(&out, &json, false, false, false); + + out << Endl; + return out.Str(); +} + +TBillRecord::TUsage TBillRecord::RequestUnits(ui64 quantity, TInstant start, TInstant finish) { + return TUsage() + .Type(TUsage::EType::Delta) + .Unit(TUsage::EUnit::RequestUnit) + .Quantity(quantity) + .Start(start) + .Finish(finish); +} + +TBillRecord::TUsage TBillRecord::RequestUnits(ui64 quantity, TInstant now) { + return RequestUnits(quantity, now, now); +} + } // NKikimr diff --git a/ydb/core/metering/bill_record.h b/ydb/core/metering/bill_record.h index 0c11f708ef5..2959f7f29bd 100644 --- a/ydb/core/metering/bill_record.h +++ b/ydb/core/metering/bill_record.h @@ -1,70 +1,70 @@ -#pragma once - -#include <library/cpp/json/json_value.h> - -#include <util/datetime/base.h> - -namespace NKikimr { - -struct TBillRecord { -#define BILL_RECORD_FIELD_SETTER(type, name) \ - TSelf& name(const type& value) { \ - name##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define BILL_RECORD_FIELD(type, name) \ - BILL_RECORD_FIELD_SETTER(type, name) \ - type name##_ - -#define BILL_RECORD_FIELD_DEFAULT(type, name, defaultValue) \ - BILL_RECORD_FIELD_SETTER(type, name) \ - type name##_ = defaultValue - - using TSelf = TBillRecord; - - struct TUsage { - using TSelf = TUsage; - - enum class EType { - Delta /* "delta" */, - }; - - enum class EUnit { - RequestUnit /* "request_unit" */, - }; - - BILL_RECORD_FIELD(EType, Type); - BILL_RECORD_FIELD(EUnit, Unit); - BILL_RECORD_FIELD(ui64, Quantity); - BILL_RECORD_FIELD(TInstant, Start); - BILL_RECORD_FIELD(TInstant, Finish); - - NJson::TJsonMap ToJson() const; - TString ToString() const; - - }; // TUsage - - BILL_RECORD_FIELD_DEFAULT(TString, Version, "1.0.0"); - BILL_RECORD_FIELD(TString, Id); - BILL_RECORD_FIELD_DEFAULT(TString, Schema, "ydb.serverless.requests.v1"); - BILL_RECORD_FIELD(TString, CloudId); - BILL_RECORD_FIELD(TString, FolderId); - BILL_RECORD_FIELD(TString, ResourceId); - BILL_RECORD_FIELD_DEFAULT(TString, SourceId, "sless-docapi-ydb-ss"); - BILL_RECORD_FIELD(TInstant, SourceWt); - BILL_RECORD_FIELD_DEFAULT(NJson::TJsonMap, Tags, {}); - BILL_RECORD_FIELD(TUsage, Usage); - - NJson::TJsonMap ToJson() const; - TString ToString() const; - - static TUsage RequestUnits(ui64 quantity, TInstant start, TInstant finish); - static TUsage RequestUnits(ui64 quantity, TInstant now); - -#undef BILL_RECORD_FIELD_DEFAULT -#undef BILL_RECORD_FIELD -#undef BILL_RECORD_FIELD_SETTER +#pragma once + +#include <library/cpp/json/json_value.h> + +#include <util/datetime/base.h> + +namespace NKikimr { + +struct TBillRecord { +#define BILL_RECORD_FIELD_SETTER(type, name) \ + TSelf& name(const type& value) { \ + name##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define BILL_RECORD_FIELD(type, name) \ + BILL_RECORD_FIELD_SETTER(type, name) \ + type name##_ + +#define BILL_RECORD_FIELD_DEFAULT(type, name, defaultValue) \ + BILL_RECORD_FIELD_SETTER(type, name) \ + type name##_ = defaultValue + + using TSelf = TBillRecord; + + struct TUsage { + using TSelf = TUsage; + + enum class EType { + Delta /* "delta" */, + }; + + enum class EUnit { + RequestUnit /* "request_unit" */, + }; + + BILL_RECORD_FIELD(EType, Type); + BILL_RECORD_FIELD(EUnit, Unit); + BILL_RECORD_FIELD(ui64, Quantity); + BILL_RECORD_FIELD(TInstant, Start); + BILL_RECORD_FIELD(TInstant, Finish); + + NJson::TJsonMap ToJson() const; + TString ToString() const; + + }; // TUsage + + BILL_RECORD_FIELD_DEFAULT(TString, Version, "1.0.0"); + BILL_RECORD_FIELD(TString, Id); + BILL_RECORD_FIELD_DEFAULT(TString, Schema, "ydb.serverless.requests.v1"); + BILL_RECORD_FIELD(TString, CloudId); + BILL_RECORD_FIELD(TString, FolderId); + BILL_RECORD_FIELD(TString, ResourceId); + BILL_RECORD_FIELD_DEFAULT(TString, SourceId, "sless-docapi-ydb-ss"); + BILL_RECORD_FIELD(TInstant, SourceWt); + BILL_RECORD_FIELD_DEFAULT(NJson::TJsonMap, Tags, {}); + BILL_RECORD_FIELD(TUsage, Usage); + + NJson::TJsonMap ToJson() const; + TString ToString() const; + + static TUsage RequestUnits(ui64 quantity, TInstant start, TInstant finish); + static TUsage RequestUnits(ui64 quantity, TInstant now); + +#undef BILL_RECORD_FIELD_DEFAULT +#undef BILL_RECORD_FIELD +#undef BILL_RECORD_FIELD_SETTER }; - + } // NKikimr diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp index f1046f5d933..ba9b50cde00 100644 --- a/ydb/core/mind/bscontroller/sys_view.cpp +++ b/ydb/core/mind/bscontroller/sys_view.cpp @@ -522,7 +522,7 @@ void CopyInfo(TDstMap& dst, TDeletedSet& deleted, const TSrcMap& src, TChangedSe } void TBlobStorageController::UpdateSystemViews() { - if (!AppData()->FeatureFlags.GetEnableSystemViews()) { + if (!AppData()->FeatureFlags.GetEnableSystemViews()) { return; } diff --git a/ydb/core/mind/configured_tablet_bootstrapper.cpp b/ydb/core/mind/configured_tablet_bootstrapper.cpp index 8b57153042d..008acd2b400 100644 --- a/ydb/core/mind/configured_tablet_bootstrapper.cpp +++ b/ydb/core/mind/configured_tablet_bootstrapper.cpp @@ -11,7 +11,7 @@ #include <ydb/core/tx/mediator/mediator.h> #include <ydb/core/tx/tx_allocator/txallocator.h> #include <ydb/core/tx/datashard/datashard.h> -#include <ydb/core/tx/replication/controller/controller.h> +#include <ydb/core/tx/replication/controller/controller.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/sequenceshard/sequenceshard.h> #include <ydb/core/keyvalue/keyvalue.h> @@ -220,9 +220,9 @@ TIntrusivePtr<TTabletSetupInfo> MakeTabletSetupInfo( case TTabletTypes::SequenceShard: createFunc = &NSequenceShard::CreateSequenceShard; break; - case TTabletTypes::ReplicationController: - createFunc = &NReplication::CreateController; - break; + case TTabletTypes::ReplicationController: + createFunc = &NReplication::CreateController; + break; default: return nullptr; } diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index cf81259001a..7e9a83b354e 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -1092,8 +1092,8 @@ public: return "TS"; case TTabletTypes::SequenceShard: return "S"; - case TTabletTypes::ReplicationController: - return "RC"; + case TTabletTypes::ReplicationController: + return "RC"; default: return Sprintf("%d", (int)type); } diff --git a/ydb/core/mind/lease_holder.cpp b/ydb/core/mind/lease_holder.cpp index 1d96b707613..5d47d04c71f 100644 --- a/ydb/core/mind/lease_holder.cpp +++ b/ydb/core/mind/lease_holder.cpp @@ -232,7 +232,7 @@ private: void StopNode(const TActorContext &ctx) { - LOG_ERROR_S(ctx, NKikimrServices::NODE_BROKER, "Stop node upon lease expiration (exit code 2)"); + LOG_ERROR_S(ctx, NKikimrServices::NODE_BROKER, "Stop node upon lease expiration (exit code 2)"); AppData(ctx)->KikimrShouldContinue->ShouldStop(2); } diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index 3b11e55f016..09fc15f76dd 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -945,9 +945,9 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { }; struct TTenantInfo { - TTenantInfo(const TRegistrationInfo &info, const TSubDomainKey &domainKey) + TTenantInfo(const TRegistrationInfo &info, const TSubDomainKey &domainKey) : Info(info) - , DomainKey(domainKey) + , DomainKey(domainKey) {} TTenantInfo(const TTenantInfo &other) = default; @@ -958,7 +958,7 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { TActorId Subscriber; TVector<TTabletId> HiveIds; THashMap<TString, TString> Attributes; - TSubDomainKey DomainKey; + TSubDomainKey DomainKey; }; TString Domain; @@ -1006,8 +1006,8 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { ev = new TEvLocal::TEvTenantStatus(tenant, TEvLocal::TEvTenantStatus::STARTED, it->second.Info.ResourceLimit, - it->second.Attributes, - it->second.DomainKey); + it->second.Attributes, + it->second.DomainKey); else ev = new TEvLocal::TEvTenantStatus(tenant, @@ -1064,10 +1064,10 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { const TActorContext &ctx) { Y_VERIFY(!RunningTenants.contains(info.TenantName)); - const auto domainKey = TSubDomainKey(SchemeRoot, 1); - RunningTenants.emplace(std::make_pair(info.TenantName, TTenantInfo(info, domainKey))); + const auto domainKey = TSubDomainKey(SchemeRoot, 1); + RunningTenants.emplace(std::make_pair(info.TenantName, TTenantInfo(info, domainKey))); for (auto id : HiveIds) { - RegisterLocalNode(info.TenantName, info.ResourceLimit, id, {domainKey}, ctx); + RegisterLocalNode(info.TenantName, info.ResourceLimit, id, {domainKey}, ctx); LOG_DEBUG_S(ctx, NKikimrServices::LOCAL, LogPrefix << "Binding to hive " << id << @@ -1082,10 +1082,10 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { const TActorContext &ctx) { const auto &domainDesc = rec.GetPathDescription().GetDomainDescription(); - const auto domainKey = TSubDomainKey(domainDesc.GetDomainKey()); + const auto domainKey = TSubDomainKey(domainDesc.GetDomainKey()); Y_VERIFY(!RunningTenants.contains(task.Info.TenantName)); - TTenantInfo info(task.Info, domainKey); + TTenantInfo info(task.Info, domainKey); for (auto &attr : rec.GetPathDescription().GetUserAttributes()) info.Attributes.emplace(std::make_pair(attr.GetKey(), attr.GetValue())); RunningTenants.emplace(std::make_pair(task.Info.TenantName, info)); @@ -1097,7 +1097,7 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> { LogPrefix << "Binding tenant " << task.Info.TenantName << " to hive " << hId << " (allocated resources: " << task.Info.ResourceLimit.ShortDebugString() << ")"); - RegisterLocalNode(task.Info.TenantName, task.Info.ResourceLimit, hId, {domainKey}, ctx); + RegisterLocalNode(task.Info.TenantName, task.Info.ResourceLimit, hId, {domainKey}, ctx); } } diff --git a/ydb/core/mind/local.h b/ydb/core/mind/local.h index e403ccc4504..05c9bdc48eb 100644 --- a/ydb/core/mind/local.h +++ b/ydb/core/mind/local.h @@ -314,7 +314,7 @@ struct TEvLocal { NKikimrTabletBase::TMetrics ResourceLimit; TString Error; THashMap<TString, TString> Attributes; - TSubDomainKey DomainKey; + TSubDomainKey DomainKey; TEvTenantStatus(const TString &tenant, EStatus status) : TenantName(tenant) @@ -330,13 +330,13 @@ struct TEvLocal { TEvTenantStatus(const TString &tenant, EStatus status, NKikimrTabletBase::TMetrics resourceLimit, - const THashMap<TString, TString> &attrs, - const TSubDomainKey &domainKey) + const THashMap<TString, TString> &attrs, + const TSubDomainKey &domainKey) : TenantName(tenant) , Status(status) , ResourceLimit(resourceLimit) , Attributes(attrs) - , DomainKey(domainKey) + , DomainKey(domainKey) {} }; diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp index daf83377c13..a2ef9383691 100644 --- a/ydb/core/mind/node_broker.cpp +++ b/ydb/core/mind/node_broker.cpp @@ -742,22 +742,22 @@ void TNodeBroker::DbUpdateNodeLease(const TNodeInfo &node, .Update<Schema::Nodes::Expire>(Epoch.NextEnd.GetValue()); } -void TNodeBroker::DbUpdateNodeLocation(const TNodeInfo &node, - TTransactionContext &txc) -{ - LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, - "Update node " << node.IdString() << " location in database" +void TNodeBroker::DbUpdateNodeLocation(const TNodeInfo &node, + TTransactionContext &txc) +{ + LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, + "Update node " << node.IdString() << " location in database" << " location=" << node.Location.ToString()); - - NIceDb::TNiceDb db(txc.DB); + + NIceDb::TNiceDb db(txc.DB); using T = Schema::Nodes; db.Table<T>().Key(node.NodeId).Update<T::Location>(node.Location.GetSerializedLocation()); // to be removed const auto& x = node.Location.GetLegacyValue(); db.Table<T>().Key(node.NodeId).Update<T::DataCenter, T::Room, T::Rack, T::Body>(x.DataCenter, x.Room, x.Rack, x.Body); -} - +} + void TNodeBroker::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { @@ -823,9 +823,9 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvResolveNode::TPtr &ev, void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev, const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvNodeBroker::TEvRegistrationRequest" - << ": request# " << ev->Get()->Record.ShortDebugString()); - + LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvNodeBroker::TEvRegistrationRequest" + << ": request# " << ev->Get()->Record.ShortDebugString()); + class TRegisterNodeActor : public TActorBootstrapped<TRegisterNodeActor> { TEvNodeBroker::TEvRegistrationRequest::TPtr Ev; TNodeBroker *Self; @@ -845,14 +845,14 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev, Become(&TThis::StateFunc); auto& record = Ev->Get()->Record; - + if (record.HasPath()) { auto req = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); auto& rset = req->ResultSet; rset.emplace_back(); auto& item = rset.back(); item.Path = NKikimr::SplitPath(record.GetPath()); - item.RedirectRequired = false; + item.RedirectRequired = false; item.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(req), IEventHandle::FlagTrackDelivery, 0); } else { @@ -865,18 +865,18 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev, auto& rset = navigate->ResultSet; Y_VERIFY(rset.size() == 1); auto& response = rset.front(); - - LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": response# " << response.ToString(*AppData()->TypeRegistry)); - - if (response.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok && response.DomainInfo) { - ScopeId = {response.DomainInfo->DomainKey.OwnerId, response.DomainInfo->DomainKey.LocalPathId}; - } else { - LOG_WARN_S(ctx, NKikimrServices::NODE_BROKER, "Cannot resolve scope id" - << ": request# " << Ev->Get()->Record.ShortDebugString() - << ", response# " << response.ToString(*AppData()->TypeRegistry)); + + LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": response# " << response.ToString(*AppData()->TypeRegistry)); + + if (response.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok && response.DomainInfo) { + ScopeId = {response.DomainInfo->DomainKey.OwnerId, response.DomainInfo->DomainKey.LocalPathId}; + } else { + LOG_WARN_S(ctx, NKikimrServices::NODE_BROKER, "Cannot resolve scope id" + << ": request# " << Ev->Get()->Record.ShortDebugString() + << ", response# " << response.ToString(*AppData()->TypeRegistry)); } - + Finish(ctx); } @@ -885,10 +885,10 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev, } void Finish(const TActorContext& ctx) { - LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Finished resolving scope id" - << ": request# " << Ev->Get()->Record.ShortDebugString() - << ": scope id# " << ScopeIdToString(ScopeId)); - + LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Finished resolving scope id" + << ": request# " << Ev->Get()->Record.ShortDebugString() + << ": scope id# " << ScopeIdToString(ScopeId)); + Self->ProcessTx(Self->CreateTxRegisterNode(Ev, ScopeId), ctx); Die(ctx); } diff --git a/ydb/core/mind/node_broker__register_node.cpp b/ydb/core/mind/node_broker__register_node.cpp index cbf9a161108..b656297837b 100644 --- a/ydb/core/mind/node_broker__register_node.cpp +++ b/ydb/core/mind/node_broker__register_node.cpp @@ -47,19 +47,19 @@ public: LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxRegisterNode Execute"); LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, "Registration request from " << host << ":" << port << " " - << (rec.GetFixedNodeId() ? "(fixed)" : "(not fixed)") << " " - << "tenant: " << (rec.HasPath() ? rec.GetPath() : "<unspecified>")); + << (rec.GetFixedNodeId() ? "(fixed)" : "(not fixed)") << " " + << "tenant: " << (rec.HasPath() ? rec.GetPath() : "<unspecified>")); TNodeLocation loc(rec.GetLocation()); Response = new TEvNodeBroker::TEvRegistrationResponse; - if (rec.HasPath() && ScopeId == NActors::TScopeId()) { - return Error(TStatus::ERROR, - TStringBuilder() << "Cannot resolve scope id for path" << rec.GetPath(), - ctx); - } - + if (rec.HasPath() && ScopeId == NActors::TScopeId()) { + return Error(TStatus::ERROR, + TStringBuilder() << "Cannot resolve scope id for path" << rec.GetPath(), + ctx); + } + // Already registered? auto it = Self->Hosts.find(std::make_tuple(host, addr, port)); if (it != Self->Hosts.end()) { @@ -82,7 +82,7 @@ public: node.Location = loc; Self->DbUpdateNodeLocation(node, txc); node.LegacyUpdatePending = false; - } + } if (!node.IsFixed() && rec.GetFixedNodeId()) { Self->DbFixNodeId(node, txc); diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h index f937a5eae5a..e3b68b0587b 100644 --- a/ydb/core/mind/node_broker_impl.h +++ b/ydb/core/mind/node_broker_impl.h @@ -262,8 +262,8 @@ private: TTransactionContext &txc); void DbUpdateNodeLease(const TNodeInfo &node, TTransactionContext &txc); - void DbUpdateNodeLocation(const TNodeInfo &node, - TTransactionContext &txc); + void DbUpdateNodeLocation(const TNodeInfo &node, + TTransactionContext &txc); void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/mind/tenant_pool.cpp b/ydb/core/mind/tenant_pool.cpp index b540c28b84a..80236570b9e 100644 --- a/ydb/core/mind/tenant_pool.cpp +++ b/ydb/core/mind/tenant_pool.cpp @@ -43,13 +43,13 @@ struct TDynamicSlotInfo : public TThrRefBase { struct TTenantInfo : public TThrRefBase { using TPtr = TIntrusivePtr<TTenantInfo>; - using EState = NKikimrTenantPool::EState; + using EState = NKikimrTenantPool::EState; TTenantInfo(const TString &name) : Name(name) , HasStaticSlot(false) , LastStatus(TEvLocal::TEvTenantStatus::STOPPED) - , State(EState::TENANT_ASSIGNED) + , State(EState::TENANT_ASSIGNED) { } @@ -103,26 +103,26 @@ struct TTenantInfo : public TThrRefBase { NKikimrTabletBase::TMetrics StaticResourceLimit; NKikimrTabletBase::TMetrics ResourceLimit; TEvLocal::TEvTenantStatus::EStatus LastStatus; - EState State; + EState State; THashMap<TString, TString> Attributes; - TSubDomainKey DomainKey; + TSubDomainKey DomainKey; }; -struct TTenantSlotBrokerInfo { - ui64 TabletId = 0; - ui64 Generation = 0; - ui64 SeqNo = 0; +struct TTenantSlotBrokerInfo { + ui64 TabletId = 0; + ui64 Generation = 0; + ui64 SeqNo = 0; TActorId ActorId; TActorId Pipe; -}; - +}; + class TDomainTenantPool : public TActorBootstrapped<TDomainTenantPool> { using TActorBase = TActorBootstrapped<TDomainTenantPool>; TString DomainName; TString LogPrefix; TActorId LocalID; - TTenantSlotBrokerInfo TenantSlotBroker; + TTenantSlotBrokerInfo TenantSlotBroker; TTenantPoolConfig::TPtr Config; THashMap<TString, TDynamicSlotInfo::TPtr> DynamicSlots; THashMap<TString, TTenantInfo::TPtr> Tenants; @@ -152,7 +152,7 @@ public: void Die(const TActorContext &ctx) override { - NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); + NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); ctx.Send(LocalID, new TEvents::TEvPoisonPill); TActorBase::Die(ctx); @@ -160,28 +160,28 @@ public: void TryToRegister(const TActorContext &ctx) { - Y_VERIFY(!TenantSlotBroker.Pipe); + Y_VERIFY(!TenantSlotBroker.Pipe); NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = { .MinRetryTime = TDuration::MilliSeconds(10), .MaxRetryTime = TDuration::Seconds(1), }; - auto pipe = NTabletPipe::CreateClient(ctx.SelfID, TenantSlotBroker.TabletId, pipeConfig); - TenantSlotBroker.Pipe = ctx.ExecutorThread.RegisterActor(pipe); + auto pipe = NTabletPipe::CreateClient(ctx.SelfID, TenantSlotBroker.TabletId, pipeConfig); + TenantSlotBroker.Pipe = ctx.ExecutorThread.RegisterActor(pipe); auto request = MakeHolder<TEvTenantSlotBroker::TEvRegisterPool>(); ActorIdToProto(TenantSlotBroker.Pipe, request->Record.MutableClientId()); - request->Record.SetSeqNo(++TenantSlotBroker.SeqNo); - NTabletPipe::SendData(ctx, TenantSlotBroker.Pipe, request.Release()); + request->Record.SetSeqNo(++TenantSlotBroker.SeqNo); + NTabletPipe::SendData(ctx, TenantSlotBroker.Pipe, request.Release()); LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, LogPrefix << "try to register in tenant slot broker (pipe " - << TenantSlotBroker.Pipe << ")"); + << TenantSlotBroker.Pipe << ")"); } void HandlePipeDestroyed(const TActorContext &ctx) { - NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); + NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); TryToRegister(ctx); } @@ -205,8 +205,8 @@ public: attr.SetKey(pr.first); attr.SetValue(pr.second); } - status.SetState(slot->AssignedTenant->State); - *status.MutableDomainKey() = NKikimrSubDomains::TDomainKey(slot->AssignedTenant->DomainKey); + status.SetState(slot->AssignedTenant->State); + *status.MutableDomainKey() = NKikimrSubDomains::TDomainKey(slot->AssignedTenant->DomainKey); } status.MutableResourceLimit()->CopyFrom(slot->ResourceLimit); } @@ -361,8 +361,8 @@ public: attr.SetKey(pr.first); attr.SetValue(pr.second); } - slotStatus.SetState(it->second->State); - *slotStatus.MutableDomainKey() = NKikimrSubDomains::TDomainKey(it->second->DomainKey); + slotStatus.SetState(it->second->State); + *slotStatus.MutableDomainKey() = NKikimrSubDomains::TDomainKey(it->second->DomainKey); } } } @@ -413,7 +413,7 @@ public: auto domain = AppData(ctx)->DomainsInfo->GetDomainByName(DomainName); Y_VERIFY(domain); - TenantSlotBroker.TabletId = MakeTenantSlotBrokerID(domain->DefaultStateStorageGroup); + TenantSlotBroker.TabletId = MakeTenantSlotBrokerID(domain->DefaultStateStorageGroup); for (auto &pr : Config->StaticSlots) { TTenantInfo::TPtr tenant = new TTenantInfo(pr.second.GetTenantName()); @@ -535,14 +535,14 @@ public: Y_VERIFY(!tenant->HasStaticSlot || ev->Get()->Status == TEvLocal::TEvTenantStatus::STARTED, "Cannot start static tenant %s: %s", ev->Get()->TenantName.data(), ev->Get()->Error.data()); - bool modified = false; - bool processPendingActions = false; - TTenantInfo::EState state = tenant->State; + bool modified = false; + bool processPendingActions = false; + TTenantInfo::EState state = tenant->State; tenant->LastStatus = ev->Get()->Status; THashSet<TDynamicSlotInfo::TPtr, TPtrHash> toProcess = tenant->AssignedSlots; switch(ev->Get()->Status) { case TEvLocal::TEvTenantStatus::STARTED: - state = TTenantInfo::EState::TENANT_OK; + state = TTenantInfo::EState::TENANT_OK; if (!tenant->HasStaticSlot && !tenant->AssignedSlots.size()) { LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, LogPrefix << "started tenant " << tenant->Name @@ -558,18 +558,18 @@ public: LogPrefix << "started tenant " << tenant->Name); if (tenant->Attributes != ev->Get()->Attributes) { tenant->Attributes = std::move(ev->Get()->Attributes); - modified = true; + modified = true; + } + if (tenant->DomainKey != ev->Get()->DomainKey) { + tenant->DomainKey = ev->Get()->DomainKey; + modified = true; } - if (tenant->DomainKey != ev->Get()->DomainKey) { - tenant->DomainKey = ev->Get()->DomainKey; - modified = true; - } SendTenantStatus(tenant, NKikimrTenantPool::SUCCESS, ctx); - processPendingActions = true; + processPendingActions = true; } break; case TEvLocal::TEvTenantStatus::STOPPED: - state = TTenantInfo::EState::TENANT_OK; + state = TTenantInfo::EState::TENANT_OK; if (tenant->AssignedSlots.size()) { LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, LogPrefix << "stopped tenant " << tenant->Name @@ -581,31 +581,31 @@ public: } break; case TEvLocal::TEvTenantStatus::UNKNOWN_TENANT: - modified = true; - state = TTenantInfo::EState::TENANT_UNKNOWN; + modified = true; + state = TTenantInfo::EState::TENANT_UNKNOWN; LOG_ERROR_S(ctx, NKikimrServices::TENANT_POOL, LogPrefix << "couldn't start unknown tenant " << tenant->Name); SendTenantStatus(tenant, NKikimrTenantPool::UNKNOWN_TENANT, ev->Get()->Error, ctx); DetachAllSlots(tenant, ctx); - processPendingActions = true; - break; - } - - if (modified || tenant->State != state) { - tenant->State = state; + processPendingActions = true; + break; + } + + if (modified || tenant->State != state) { + tenant->State = state; SendStatusUpdates(ctx); - } - - if (processPendingActions) { + } + + if (processPendingActions) { ProcessPendingActions(toProcess, ctx); } } void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { TEvTabletPipe::TEvClientConnected *msg = ev->Get(); - if (msg->ClientId != TenantSlotBroker.Pipe) + if (msg->ClientId != TenantSlotBroker.Pipe) return; if (msg->Status == NKikimrProto::OK) return; @@ -614,7 +614,7 @@ public: void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx) { TEvTabletPipe::TEvClientDestroyed *msg = ev->Get(); - if (msg->ClientId != TenantSlotBroker.Pipe) + if (msg->ClientId != TenantSlotBroker.Pipe) return; HandlePipeDestroyed(ctx); } @@ -634,7 +634,7 @@ public: LogPrefix << ev->Sender << " configures slot '" << rec.GetSlotId() << "' for tenant '" << rec.GetAssignedTenant() << "'"); - if (ev->Sender != TenantSlotBroker.ActorId) { + if (ev->Sender != TenantSlotBroker.ActorId) { LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, LogPrefix << "configure sender " << ev->Sender << " doesn't own pool"); SendConfigureError(ev, rec.GetSlotId(), NKikimrTenantPool::NOT_OWNER, @@ -662,37 +662,37 @@ public: void Handle(TEvTenantPool::TEvTakeOwnership::TPtr &ev, const TActorContext &ctx) { - const auto& record = ev->Get()->Record; - const bool isOldStyle = !record.HasGeneration() && !record.HasSeqNo(); - const ui64 generation = isOldStyle ? 0 : record.GetGeneration(); - const ui64 seqNo = isOldStyle ? 0 : record.GetSeqNo(); - - if (isOldStyle || generation > TenantSlotBroker.Generation || seqNo == TenantSlotBroker.SeqNo) { - if (seqNo == Max<ui64>()) { - NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); - TryToRegister(ctx); - - return; - } else if (!isOldStyle && seqNo != TenantSlotBroker.SeqNo) { - return; - } - - TenantSlotBroker.Generation = generation; - - if (TenantSlotBroker.ActorId != ev->Sender) { - if (TenantSlotBroker.ActorId) { - LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, - LogPrefix << TenantSlotBroker.ActorId << " lost ownership"); - ctx.Send(TenantSlotBroker.ActorId, new TEvTenantPool::TEvLostOwnership); - } - - TenantSlotBroker.ActorId = ev->Sender; + const auto& record = ev->Get()->Record; + const bool isOldStyle = !record.HasGeneration() && !record.HasSeqNo(); + const ui64 generation = isOldStyle ? 0 : record.GetGeneration(); + const ui64 seqNo = isOldStyle ? 0 : record.GetSeqNo(); + + if (isOldStyle || generation > TenantSlotBroker.Generation || seqNo == TenantSlotBroker.SeqNo) { + if (seqNo == Max<ui64>()) { + NTabletPipe::CloseAndForgetClient(SelfId(), TenantSlotBroker.Pipe); + TryToRegister(ctx); + + return; + } else if (!isOldStyle && seqNo != TenantSlotBroker.SeqNo) { + return; + } + + TenantSlotBroker.Generation = generation; + + if (TenantSlotBroker.ActorId != ev->Sender) { + if (TenantSlotBroker.ActorId) { + LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, + LogPrefix << TenantSlotBroker.ActorId << " lost ownership"); + ctx.Send(TenantSlotBroker.ActorId, new TEvTenantPool::TEvLostOwnership); + } + + TenantSlotBroker.ActorId = ev->Sender; LOG_DEBUG_S(ctx, NKikimrServices::TENANT_POOL, - LogPrefix << TenantSlotBroker.ActorId << " took ownership"); + LogPrefix << TenantSlotBroker.ActorId << " took ownership"); } - auto event = BuildStatusEvent(); - ctx.Send(ev->Sender, std::move(event), 0, ev->Cookie); + auto event = BuildStatusEvent(); + ctx.Send(ev->Sender, std::move(event), 0, ev->Cookie); } } @@ -711,14 +711,14 @@ public: PRE() { str << "Tenant pool for " << DomainName << Endl << "LocalID: " << LocalID << Endl - << "TenantSlotBroker:" << Endl - << " TabletId: " << TenantSlotBroker.TabletId << Endl - << " Generation: " << TenantSlotBroker.Generation << Endl - << " ActorId: " << TenantSlotBroker.ActorId << Endl << Endl; + << "TenantSlotBroker:" << Endl + << " TabletId: " << TenantSlotBroker.TabletId << Endl + << " Generation: " << TenantSlotBroker.Generation << Endl + << " ActorId: " << TenantSlotBroker.ActorId << Endl << Endl; str << "Config:" << Endl << " IsEnabled: " << Config->IsEnabled << Endl - << " NodeType: " << Config->NodeType << Endl + << " NodeType: " << Config->NodeType << Endl << " StaticSlotLabel: " << Config->StaticSlotLabel << Endl << " DynamicSlotLabel: " << Config->DynamicSlotLabel << Endl << " StaticSlots:" << Endl; @@ -747,9 +747,9 @@ public: << " HasStaticSlot: " << pr.second->HasStaticSlot << Endl << " StaticResourceLimit: " << pr.second->StaticResourceLimit.ShortDebugString() << Endl << " ResourceLimit: " << pr.second->ResourceLimit.ShortDebugString() << Endl - << " LastStatus: " << (int)pr.second->LastStatus << Endl - << " State: " << (int)pr.second->State << Endl - << " DomainKey: " << pr.second->DomainKey << Endl; + << " LastStatus: " << (int)pr.second->LastStatus << Endl + << " State: " << (int)pr.second->State << Endl + << " DomainKey: " << pr.second->DomainKey << Endl; str << " AssignedSlots:" << Endl; for (auto &slot : pr.second->AssignedSlots) diff --git a/ydb/core/mind/tenant_pool.h b/ydb/core/mind/tenant_pool.h index f7c01094604..0202ba78634 100644 --- a/ydb/core/mind/tenant_pool.h +++ b/ydb/core/mind/tenant_pool.h @@ -46,15 +46,15 @@ struct TEvTenantPool { struct TEvConfigureSlotResult : public TEventPB<TEvConfigureSlotResult, NKikimrTenantPool::TConfigureSlotResult, EvConfigureSlotResult> {}; - struct TEvTakeOwnership : public TEventPB<TEvTakeOwnership, NKikimrTenantPool::TTakeOwnership, EvTakeOwnership> { - TEvTakeOwnership() = default; - - TEvTakeOwnership(ui64 generation, ui64 seqNo = Max<ui64>()) { - Record.SetGeneration(generation); - Record.SetSeqNo(seqNo); - } - }; - + struct TEvTakeOwnership : public TEventPB<TEvTakeOwnership, NKikimrTenantPool::TTakeOwnership, EvTakeOwnership> { + TEvTakeOwnership() = default; + + TEvTakeOwnership(ui64 generation, ui64 seqNo = Max<ui64>()) { + Record.SetGeneration(generation); + Record.SetSeqNo(seqNo); + } + }; + struct TEvLostOwnership : public TEventPB<TEvLostOwnership, NKikimrTenantPool::TLostOwnership, EvLostOwnership> {}; }; diff --git a/ydb/core/mind/tenant_slot_broker.cpp b/ydb/core/mind/tenant_slot_broker.cpp index aa38f9604df..65f7a86328b 100644 --- a/ydb/core/mind/tenant_slot_broker.cpp +++ b/ydb/core/mind/tenant_slot_broker.cpp @@ -398,10 +398,10 @@ TTenantSlotBroker::TSlot::TPtr TTenantSlotBroker::TFreeSlotsIndex::Find(const TS return *it2->second.begin(); } -ui64 TTenantSlotBroker::Generation() const { - return Executor()->Generation(); -} - +ui64 TTenantSlotBroker::Generation() const { + return Executor()->Generation(); +} + void TTenantSlotBroker::OnActivateExecutor(const TActorContext &ctx) { RequestId = Now().GetValue(); @@ -1692,7 +1692,7 @@ void TTenantSlotBroker::Handle(TEvTenantPool::TEvLostOwnership::TPtr &ev, LOG_DEBUG_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, "Re-taking ownership of tenant pool on node " << nodeId); - ctx.Send(MakeTenantPoolID(nodeId, DomainId), new TEvTenantPool::TEvTakeOwnership(Generation())); + ctx.Send(MakeTenantPoolID(nodeId, DomainId), new TEvTenantPool::TEvTakeOwnership(Generation())); } void TTenantSlotBroker::Handle(TEvTabletPipe::TEvServerConnected::TPtr &ev, @@ -1769,14 +1769,14 @@ void TTenantSlotBroker::Handle(TEvTenantSlotBroker::TEvListTenants::TPtr &ev, void TTenantSlotBroker::Handle(TEvTenantSlotBroker::TEvRegisterPool::TPtr &ev, const TActorContext &ctx) { - const auto &record = ev->Get()->Record; + const auto &record = ev->Get()->Record; auto nodeId = ev->Sender.NodeId(); KnownPoolPipes.insert(ActorIdFromProto(record.GetClientId())); *Counters->ConnectedPools = KnownPoolPipes.size(); DisconnectNodeSlots(nodeId, ctx); - ctx.Send(ev->Sender, new TEvTenantPool::TEvTakeOwnership(Generation(), record.GetSeqNo())); + ctx.Send(ev->Sender, new TEvTenantPool::TEvTakeOwnership(Generation(), record.GetSeqNo())); } IActor *CreateTenantSlotBroker(const TActorId &tablet, diff --git a/ydb/core/mind/tenant_slot_broker__load_state.cpp b/ydb/core/mind/tenant_slot_broker__load_state.cpp index 78321f9c17c..345b131ea3c 100644 --- a/ydb/core/mind/tenant_slot_broker__load_state.cpp +++ b/ydb/core/mind/tenant_slot_broker__load_state.cpp @@ -171,14 +171,14 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, "Taking ownership of tenant pool on node " << pr.first); - ctx.Send(MakeTenantPoolID(pr.first, Self->DomainId), new TEvTenantPool::TEvTakeOwnership(Self->Generation())); + ctx.Send(MakeTenantPoolID(pr.first, Self->DomainId), new TEvTenantPool::TEvTakeOwnership(Self->Generation())); ctx.Send(GetNameserviceActorId(), new TEvInterconnect::TEvGetNode(pr.first)); - - for (auto& slot : pr.second) { - const ui64 randomDelay = TAppData::RandomProvider->GenRand64() % Self->PendingTimeout.GetValue(); - const TDuration pendingTimeout = Self->PendingTimeout + TDuration::FromValue(randomDelay); - ctx.Schedule(pendingTimeout, new TEvPrivate::TEvCheckSlotStatus(slot, reqId)); - } + + for (auto& slot : pr.second) { + const ui64 randomDelay = TAppData::RandomProvider->GenRand64() % Self->PendingTimeout.GetValue(); + const TDuration pendingTimeout = Self->PendingTimeout + TDuration::FromValue(randomDelay); + ctx.Schedule(pendingTimeout, new TEvPrivate::TEvCheckSlotStatus(slot, reqId)); + } } return true; diff --git a/ydb/core/mind/tenant_slot_broker__scheme.h b/ydb/core/mind/tenant_slot_broker__scheme.h index 25378f5beab..88b6ba06297 100644 --- a/ydb/core/mind/tenant_slot_broker__scheme.h +++ b/ydb/core/mind/tenant_slot_broker__scheme.h @@ -82,10 +82,10 @@ struct Schema : NIceDb::Schema { struct SlotId : Column<2, NScheme::NTypeIds::Utf8> {}; struct TenantName : Column<3, NScheme::NTypeIds::Utf8> {}; struct Label : Column<4, NScheme::NTypeIds::Utf8> {}; - struct Deadline : Column<5, NScheme::NTypeIds::Uint64> {}; + struct Deadline : Column<5, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<NodeId, SlotId>; - using TColumns = TableColumns<NodeId, SlotId, TenantName, Label, Deadline>; + using TColumns = TableColumns<NodeId, SlotId, TenantName, Label, Deadline>; }; using TTables = SchemaTables<Config, RequiredSlots, Slots, SlotLabels, SlotsAllocations, BannedSlots, PinnedSlots>; diff --git a/ydb/core/mind/tenant_slot_broker__update_slot_status.cpp b/ydb/core/mind/tenant_slot_broker__update_slot_status.cpp index 44228a9ef18..73538e38d60 100644 --- a/ydb/core/mind/tenant_slot_broker__update_slot_status.cpp +++ b/ydb/core/mind/tenant_slot_broker__update_slot_status.cpp @@ -58,7 +58,7 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, "Taking ownership of tenant pool on node " << nodeId); - ctx.Send(MakeTenantPoolID(nodeId, Self->DomainId), new TEvTenantPool::TEvTakeOwnership(Self->Generation())); + ctx.Send(MakeTenantPoolID(nodeId, Self->DomainId), new TEvTenantPool::TEvTakeOwnership(Self->Generation())); return true; } @@ -75,10 +75,10 @@ public: LOG_ERROR_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, slot->IdString(true) << " cannot be assigned to tenant: " << rec.GetError()); - Modified = false; // prevent infinite loop (attach-detach-attach...) - if (slot->AssignedTenant) { - slot->AssignedTenant->GetAllocation(slot->UsedAs)->DecPending(); - } + Modified = false; // prevent infinite loop (attach-detach-attach...) + if (slot->AssignedTenant) { + slot->AssignedTenant->GetAllocation(slot->UsedAs)->DecPending(); + } Self->DetachSlotNoConfigure(slot, txc); return true; } diff --git a/ydb/core/mind/tenant_slot_broker_impl.h b/ydb/core/mind/tenant_slot_broker_impl.h index 480a6a28fa8..b5e3b107ab5 100644 --- a/ydb/core/mind/tenant_slot_broker_impl.h +++ b/ydb/core/mind/tenant_slot_broker_impl.h @@ -14,8 +14,8 @@ #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> -#include <util/datetime/base.h> -#include <util/generic/map.h> +#include <util/datetime/base.h> +#include <util/generic/map.h> #include <util/generic/set.h> namespace NKikimr { @@ -502,14 +502,14 @@ private: struct TPinnedSlotInfo { TString TenantName; TString Label; - TInstant Deadline; + TInstant Deadline; + }; + + struct TEnqueuedUnpin { + TSlotId SlotId; + bool Scheduled; }; - struct TEnqueuedUnpin { - TSlotId SlotId; - bool Scheduled; - }; - using TSlotSet = THashSet<TSlot::TPtr, TPtrHash>; class TCollocationGroup : public TThrRefBase { @@ -912,7 +912,7 @@ public: ui64 RequestId; }; - + }; private: @@ -924,7 +924,7 @@ private: class TTxLoadState; class TTxPinSlot; class TTxUnbanSlot; - class TTxUnpinSlotImpl; + class TTxUnpinSlotImpl; class TTxUnpinSlot; class TTxUpdateConfig; class TTxUpdateNodeLocation; @@ -942,8 +942,8 @@ private: ITransaction *CreateTxUpdatePoolStatus(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev); ITransaction *CreateTxUpdateSlotStatus(TEvTenantPool::TEvConfigureSlotResult::TPtr &ev); - - ui64 Generation() const; + + ui64 Generation() const; void OnActivateExecutor(const TActorContext &ctx) override; void OnDetach(const TActorContext &ctx) override; void OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, @@ -1163,7 +1163,7 @@ private: THashMap<TSlotId, TSlot::TPtr> Slots; THashSet<TSlotId> BannedSlots; THashMap<TSlotId, TPinnedSlotInfo> PinnedSlots; - TMap<TInstant, TEnqueuedUnpin> EnqueuedUnpins; + TMap<TInstant, TEnqueuedUnpin> EnqueuedUnpins; // NodeId -> {Slots} THashMap<ui32, TSlotSet> SlotsByNodeId; TFreeSlotsIndex FreeSlots; diff --git a/ydb/core/mind/tenant_ut_pool.cpp b/ydb/core/mind/tenant_ut_pool.cpp index ff36740a901..9ac6b6843c3 100644 --- a/ydb/core/mind/tenant_ut_pool.cpp +++ b/ydb/core/mind/tenant_ut_pool.cpp @@ -1196,71 +1196,71 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) { CheckLabels(counters, TENANT1_1_NAME, "slot-1", attrs, monCfg); } - - void TestState( - const TTenantTestConfig::TStaticSlotConfig& staticSlot, - const TTenantTestConfig::TDynamicSlotConfig& dynamicSlot, - NKikimrTenantPool::EState expected) { - - TTenantTestConfig config = { - // Domains {name, schemeshard {{ subdomain_names }}} - {{ {DOMAIN1_NAME, SCHEME_SHARD1_ID, {{ TENANT1_1_NAME }}} }}, - HIVE_ID, // HiveId - true, // FakeTenantSlotBroker - true, // FakeSchemeShard - false, // CreateConsole - {{{ {}, {}, "node-type" }}}, // Nodes - 1 // DataCenterCount - }; - - if (staticSlot.Tenant) { - config.Nodes.back().TenantPoolConfig.StaticSlots.push_back(staticSlot); - } else if (dynamicSlot.Tenant) { - config.Nodes.back().TenantPoolConfig.DynamicSlots.push_back(dynamicSlot); - } - + + void TestState( + const TTenantTestConfig::TStaticSlotConfig& staticSlot, + const TTenantTestConfig::TDynamicSlotConfig& dynamicSlot, + NKikimrTenantPool::EState expected) { + + TTenantTestConfig config = { + // Domains {name, schemeshard {{ subdomain_names }}} + {{ {DOMAIN1_NAME, SCHEME_SHARD1_ID, {{ TENANT1_1_NAME }}} }}, + HIVE_ID, // HiveId + true, // FakeTenantSlotBroker + true, // FakeSchemeShard + false, // CreateConsole + {{{ {}, {}, "node-type" }}}, // Nodes + 1 // DataCenterCount + }; + + if (staticSlot.Tenant) { + config.Nodes.back().TenantPoolConfig.StaticSlots.push_back(staticSlot); + } else if (dynamicSlot.Tenant) { + config.Nodes.back().TenantPoolConfig.DynamicSlots.push_back(dynamicSlot); + } + TTenantTestRuntime runtime(config, {}, false); - + const TActorId& sender = runtime.Sender; const TActorId tenantPoolRoot = MakeTenantPoolRootID(); const TActorId tenantPool = MakeTenantPoolID(runtime.GetNodeId(0), 0); - - using TEvStatus = TEvTenantPool::TEvTenantPoolStatus; - using EState = NKikimrTenantPool::EState; - - auto checker = [](auto ev, EState expectedState) { - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetSlots(0).GetState(), expectedState); - }; - - runtime.CreateTenantPool(0); - - // Subscribe on root pool and wait until domain pool started - runtime.Send(new IEventHandle(tenantPoolRoot, sender, new TEvents::TEvSubscribe())); - checker(runtime.GrabEdgeEvent<TEvStatus>(sender), EState::TENANT_ASSIGNED); - - // Get status from domain pool - runtime.Send(new IEventHandle(tenantPool, sender, new TEvTenantPool::TEvGetStatus(true))); - checker(runtime.GrabEdgeEvent<TEvStatus>(sender), expected); - } - - Y_UNIT_TEST(TestStateStatic) { - TestState({TENANT1_1_NAME, {1, 1, 1}}, {}, NKikimrTenantPool::EState::TENANT_OK); - } - - Y_UNIT_TEST(TestStateDynamic) { - TestState({}, {DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, TENANT1_1_NAME, {1, 1, 1}}, NKikimrTenantPool::EState::TENANT_OK); - } - - Y_UNIT_TEST(TestStateDynamicTenantUnknown) { - // After unsuccessful resolve on SS, tenant will be detached from slot, so state is unknown - TestState({}, {DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, TENANT1_U_NAME, {1, 1, 1}}, NKikimrTenantPool::EState::STATE_UNKNOWN); - } + + using TEvStatus = TEvTenantPool::TEvTenantPoolStatus; + using EState = NKikimrTenantPool::EState; + + auto checker = [](auto ev, EState expectedState) { + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetSlots(0).GetState(), expectedState); + }; + + runtime.CreateTenantPool(0); + + // Subscribe on root pool and wait until domain pool started + runtime.Send(new IEventHandle(tenantPoolRoot, sender, new TEvents::TEvSubscribe())); + checker(runtime.GrabEdgeEvent<TEvStatus>(sender), EState::TENANT_ASSIGNED); + + // Get status from domain pool + runtime.Send(new IEventHandle(tenantPool, sender, new TEvTenantPool::TEvGetStatus(true))); + checker(runtime.GrabEdgeEvent<TEvStatus>(sender), expected); + } + + Y_UNIT_TEST(TestStateStatic) { + TestState({TENANT1_1_NAME, {1, 1, 1}}, {}, NKikimrTenantPool::EState::TENANT_OK); + } + + Y_UNIT_TEST(TestStateDynamic) { + TestState({}, {DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, TENANT1_1_NAME, {1, 1, 1}}, NKikimrTenantPool::EState::TENANT_OK); + } + + Y_UNIT_TEST(TestStateDynamicTenantUnknown) { + // After unsuccessful resolve on SS, tenant will be detached from slot, so state is unknown + TestState({}, {DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, TENANT1_U_NAME, {1, 1, 1}}, NKikimrTenantPool::EState::STATE_UNKNOWN); + } } } //namespace NKikimr - -template <> -void Out<NKikimrTenantPool::EState>(IOutputStream& o, NKikimrTenantPool::EState x) { - o << NKikimrTenantPool::EState_Name(x); -} + +template <> +void Out<NKikimrTenantPool::EState>(IOutputStream& o, NKikimrTenantPool::EState x) { + o << NKikimrTenantPool::EState_Name(x); +} diff --git a/ydb/core/mind/ya.make b/ydb/core/mind/ya.make index 54c1096c0e4..b9248c9d911 100644 --- a/ydb/core/mind/ya.make +++ b/ydb/core/mind/ya.make @@ -71,7 +71,7 @@ PEERDIR( ydb/core/tablet ydb/core/tablet_flat ydb/core/test_tablet - ydb/core/tx/replication/controller + ydb/core/tx/replication/controller ydb/core/tx/scheme_board ydb/core/tx/scheme_cache ydb/core/tx/schemeshard diff --git a/ydb/core/persqueue/config/config.cpp b/ydb/core/persqueue/config/config.cpp index bc2f5fbc2bd..027fc5e64a8 100644 --- a/ydb/core/persqueue/config/config.cpp +++ b/ydb/core/persqueue/config/config.cpp @@ -9,37 +9,37 @@ bool CheckPersQueueConfig(const NKikimrPQ::TPQTabletConfig& config, const bool s *error = "no PartitionConfig"; return false; } - - const auto& partitionIds = config.GetPartitionIds(); - const auto& partitions = config.GetPartitions(); - + + const auto& partitionIds = config.GetPartitionIds(); + const auto& partitions = config.GetPartitions(); + if (shouldHavePartitionsList) { - if (partitionIds.empty() && partitions.empty()) { + if (partitionIds.empty() && partitions.empty()) { if (error) *error = "empty Partitions list"; return false; } THashSet<ui32> parts; - for (const auto partitionId : partitionIds) { - if (!parts.insert(partitionId).second) { + for (const auto partitionId : partitionIds) { + if (!parts.insert(partitionId).second) { + if (error) + *error = Sprintf("duplicate partitions with id %u", partitionId); + return false; + } + } + + parts.clear(); + for (const auto& partition : partitions) { + const auto partitionId = partition.GetPartitionId(); + if (!parts.insert(partitionId).second) { if (error) - *error = Sprintf("duplicate partitions with id %u", partitionId); + *error = Sprintf("duplicate partitions with id %u", partitionId); return false; } } - - parts.clear(); - for (const auto& partition : partitions) { - const auto partitionId = partition.GetPartitionId(); - if (!parts.insert(partitionId).second) { - if (error) - *error = Sprintf("duplicate partitions with id %u", partitionId); - return false; - } - } } else { - if (!partitionIds.empty() || !partitions.empty()) { + if (!partitionIds.empty() || !partitions.empty()) { if (error) *error = "Partitions list must be empty"; return false; @@ -52,7 +52,7 @@ bool CheckPersQueueConfig(const NKikimrPQ::TPQTabletConfig& config, const bool s *error = "no lifetimeSeconds specified in TPartitionConfig"; return false; } - + return true; } diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index c84e972c486..bc892988b47 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -94,9 +94,9 @@ struct TEvPQ { EvUpdateWriteTimestamp, EvHandleWriteResponse, EvQuotaDeadlineCheck, - EvRegisterMessageGroup, - EvDeregisterMessageGroup, - EvSplitMessageGroup, + EvRegisterMessageGroup, + EvDeregisterMessageGroup, + EvSplitMessageGroup, EvUpdateCounters, EvMirrorerCounters, EvReadLimiterRequest, @@ -520,66 +520,66 @@ struct TEvPQ { TEvQuotaDeadlineCheck() {} }; - - struct TEvRegisterMessageGroup : TEventLocal<TEvRegisterMessageGroup, EvRegisterMessageGroup> { - struct TBody { - explicit TBody(const TString& sourceId, TMaybe<NKikimrPQ::TPartitionKeyRange>&& keyRange, ui64 seqNo, bool afterSplit) - : SourceId(sourceId) - , KeyRange(std::move(keyRange)) - , SeqNo(seqNo) - , AfterSplit(afterSplit) - {} - - TString SourceId; - TMaybe<NKikimrPQ::TPartitionKeyRange> KeyRange; - ui64 SeqNo; - bool AfterSplit; - TMaybe<ui64> AssignedOffset; // will be assigned upon registration - }; - - template <typename... Args> - explicit TEvRegisterMessageGroup(ui64 cookie, Args&&... args) - : Cookie(cookie) - , Body(std::forward<Args>(args)...) - {} - - const ui64 Cookie; - TBody Body; - }; - - struct TEvDeregisterMessageGroup : TEventLocal<TEvDeregisterMessageGroup, EvDeregisterMessageGroup> { - struct TBody { - explicit TBody(const TString& sourceId) - : SourceId(sourceId) - {} - - TString SourceId; - }; - - template <typename... Args> - explicit TEvDeregisterMessageGroup(ui64 cookie, Args&&... args) - : Cookie(cookie) - , Body(std::forward<Args>(args)...) - {} - - const ui64 Cookie; - TBody Body; - }; - - struct TEvSplitMessageGroup : TEventLocal<TEvSplitMessageGroup, EvSplitMessageGroup> { - using TRegisterBody = TEvRegisterMessageGroup::TBody; - using TDeregisterBody = TEvDeregisterMessageGroup::TBody; - - explicit TEvSplitMessageGroup(ui64 cookie, TVector<TDeregisterBody>&& deregistrations, TVector<TRegisterBody>&& registrations) - : Cookie(cookie) - , Deregistrations(std::move(deregistrations)) - , Registrations(std::move(registrations)) - {} - - const ui64 Cookie; - TVector<TDeregisterBody> Deregistrations; - TVector<TRegisterBody> Registrations; - }; + + struct TEvRegisterMessageGroup : TEventLocal<TEvRegisterMessageGroup, EvRegisterMessageGroup> { + struct TBody { + explicit TBody(const TString& sourceId, TMaybe<NKikimrPQ::TPartitionKeyRange>&& keyRange, ui64 seqNo, bool afterSplit) + : SourceId(sourceId) + , KeyRange(std::move(keyRange)) + , SeqNo(seqNo) + , AfterSplit(afterSplit) + {} + + TString SourceId; + TMaybe<NKikimrPQ::TPartitionKeyRange> KeyRange; + ui64 SeqNo; + bool AfterSplit; + TMaybe<ui64> AssignedOffset; // will be assigned upon registration + }; + + template <typename... Args> + explicit TEvRegisterMessageGroup(ui64 cookie, Args&&... args) + : Cookie(cookie) + , Body(std::forward<Args>(args)...) + {} + + const ui64 Cookie; + TBody Body; + }; + + struct TEvDeregisterMessageGroup : TEventLocal<TEvDeregisterMessageGroup, EvDeregisterMessageGroup> { + struct TBody { + explicit TBody(const TString& sourceId) + : SourceId(sourceId) + {} + + TString SourceId; + }; + + template <typename... Args> + explicit TEvDeregisterMessageGroup(ui64 cookie, Args&&... args) + : Cookie(cookie) + , Body(std::forward<Args>(args)...) + {} + + const ui64 Cookie; + TBody Body; + }; + + struct TEvSplitMessageGroup : TEventLocal<TEvSplitMessageGroup, EvSplitMessageGroup> { + using TRegisterBody = TEvRegisterMessageGroup::TBody; + using TDeregisterBody = TEvDeregisterMessageGroup::TBody; + + explicit TEvSplitMessageGroup(ui64 cookie, TVector<TDeregisterBody>&& deregistrations, TVector<TRegisterBody>&& registrations) + : Cookie(cookie) + , Deregistrations(std::move(deregistrations)) + , Registrations(std::move(registrations)) + {} + + const ui64 Cookie; + TVector<TDeregisterBody> Deregistrations; + TVector<TRegisterBody> Registrations; + }; struct TEvUpdateCounters : public TEventLocal<TEvUpdateCounters, EvUpdateCounters> { TEvUpdateCounters() @@ -618,6 +618,6 @@ struct TEvPQ { TEvReaderEventArrived() {} }; -}; +}; } //NKikimr diff --git a/ydb/core/persqueue/key.h b/ydb/core/persqueue/key.h index 3c5f0655661..098c67cf7d9 100644 --- a/ydb/core/persqueue/key.h +++ b/ydb/core/persqueue/key.h @@ -22,7 +22,7 @@ public: enum EMark : char { MarkUserDeprecated = 'u', MarkSourceId = 's', - MarkProtoSourceId = 'p', + MarkProtoSourceId = 'p', MarkUser = 'c' }; diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 8913c39c6f8..cc8e2419cf2 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -655,21 +655,21 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo TABLEH() {out << "SeqNo";} TABLEH() {out << "Offset";} TABLEH() {out << "WriteTimestamp";} - TABLEH() {out << "CreateTimestamp";} - TABLEH() {out << "Explicit";} - TABLEH() {out << "State";} + TABLEH() {out << "CreateTimestamp";} + TABLEH() {out << "Explicit";} + TABLEH() {out << "State";} } } TABLEBODY() { - for (const auto& [sourceId, sourceIdInfo]: SourceIdStorage.GetInMemorySourceIds()) { + for (const auto& [sourceId, sourceIdInfo]: SourceIdStorage.GetInMemorySourceIds()) { TABLER() { - TABLED() {out << EncodeHtmlPcdata(EscapeC(sourceId));} - TABLED() {out << sourceIdInfo.SeqNo;} - TABLED() {out << sourceIdInfo.Offset;} - TABLED() {out << ToStringLocalTimeUpToSeconds(sourceIdInfo.WriteTimestamp);} - TABLED() {out << ToStringLocalTimeUpToSeconds(sourceIdInfo.CreateTimestamp);} - TABLED() {out << (sourceIdInfo.Explicit ? "true" : "false");} - TABLED() {out << sourceIdInfo.State;} + TABLED() {out << EncodeHtmlPcdata(EscapeC(sourceId));} + TABLED() {out << sourceIdInfo.SeqNo;} + TABLED() {out << sourceIdInfo.Offset;} + TABLED() {out << ToStringLocalTimeUpToSeconds(sourceIdInfo.WriteTimestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(sourceIdInfo.CreateTimestamp);} + TABLED() {out << (sourceIdInfo.Explicit ? "true" : "false");} + TABLED() {out << sourceIdInfo.State;} } } } @@ -1260,11 +1260,11 @@ void TPartition::CancelAllWritesOnIdle(const TActorContext& ctx) { for (const auto& w : Requests) { ReplyError(ctx, w.GetCookie(), NPersQueue::NErrorCode::WRITE_ERROR_DISK_IS_FULL, "Disk is full"); - if (w.IsWrite()) { - const auto& msg = w.GetWrite().Msg; + if (w.IsWrite()) { + const auto& msg = w.GetWrite().Msg; Counters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); - Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); - WriteInflightSize -= msg.Data.size(); + Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); + WriteInflightSize -= msg.Data.size(); } } @@ -1288,11 +1288,11 @@ void TPartition::FailBadClient(const TActorContext& ctx) for (const auto& w : Requests) { ReplyError(ctx, w.GetCookie(), NPersQueue::NErrorCode::BAD_REQUEST, "previous write request failed"); - if (w.IsWrite()) { - const auto& msg = w.GetWrite().Msg; + if (w.IsWrite()) { + const auto& msg = w.GetWrite().Msg; Counters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); - Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); - WriteInflightSize -= msg.Data.size(); + Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); + WriteInflightSize -= msg.Data.size(); } } UpdateWriteBufferIsFullState(ctx.Now()); @@ -1300,7 +1300,7 @@ void TPartition::FailBadClient(const TActorContext& ctx) Requests.clear(); for (const auto& w : Responses) { ReplyError(ctx, w.GetCookie(), NPersQueue::NErrorCode::BAD_REQUEST, "previous write request failed"); - if (w.IsWrite()) + if (w.IsWrite()) Counters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); } Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(WriteNewSize); @@ -1395,9 +1395,9 @@ void TPartition::HandleInfoRangeRead(const NKikimrClient::TKeyValueResponse::TRe key = &pair.GetKey(); if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkSourceId) { - SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now()); - } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId) { - SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now()); + SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now()); + } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId) { + SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now()); } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser) { UsersInfoStorage.Parse(*key, pair.GetValue(), ctx); } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated) { @@ -1725,7 +1725,7 @@ void TPartition::ProcessChangeOwnerRequest(TAutoPtr<TEvPQ::TEvChangeOwner> ev, c it->second.GenerateCookie(owner, ev->PipeClient, ev->Sender, TopicName, Partition, ctx);//will change OwnerCookie //cookie is generated. but answer will be sent when all inflight writes will be done - they in the same queue 'Requests' - Requests.emplace_back(TOwnershipMsg{ev->Cookie, it->second.OwnerCookie}, WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); + Requests.emplace_back(TOwnershipMsg{ev->Cookie, it->second.OwnerCookie}, WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); Counters.Simple()[COUNTER_PQ_TABLET_RESERVED_BYTES_SIZE].Set(ReservedSize); UpdateWriteBufferIsFullState(ctx.Now()); @@ -1978,7 +1978,7 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext result.SetAvgQuotaSpeedPerHour(AvgQuotaBytes[2].GetValue()); result.SetAvgQuotaSpeedPerDay(AvgQuotaBytes[3].GetValue()); - result.SetSourceIdCount(SourceIdStorage.GetInMemorySourceIds().size()); + result.SetSourceIdCount(SourceIdStorage.GetInMemorySourceIds().size()); result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds()); result.SetWriteBytesQuota(WriteQuota.GetTotalSpeed()); @@ -2171,33 +2171,33 @@ void TPartition::ProcessUserActs(TUserInfo& userInfo, const TActorContext& ctx) } void TPartition::Handle(TEvPQ::TEvGetMaxSeqNoRequest::TPtr& ev, const TActorContext& ctx) { - auto response = MakeHolder<TEvPQ::TEvProxyResponse>(ev->Get()->Cookie); + auto response = MakeHolder<TEvPQ::TEvProxyResponse>(ev->Get()->Cookie); NKikimrClient::TResponse& resp = response->Response; resp.SetStatus(NMsgBusProxy::MSTATUS_OK); resp.SetErrorCode(NPersQueue::NErrorCode::OK); - auto& result = *resp.MutablePartitionResponse()->MutableCmdGetMaxSeqNoResult(); - for (const auto& sourceId : ev->Get()->SourceIds) { - auto& protoInfo = *result.AddSourceIdInfo(); - protoInfo.SetSourceId(sourceId); - - auto it = SourceIdStorage.GetInMemorySourceIds().find(sourceId); - if (it == SourceIdStorage.GetInMemorySourceIds().end()) { - continue; - } - - const auto& memInfo = it->second; - Y_VERIFY(memInfo.Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, memInfo.Offset); - Y_VERIFY(memInfo.SeqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, memInfo.SeqNo); - - protoInfo.SetSeqNo(memInfo.SeqNo); - protoInfo.SetOffset(memInfo.Offset); - protoInfo.SetWriteTimestampMS(memInfo.WriteTimestamp.MilliSeconds()); - protoInfo.SetExplicit(memInfo.Explicit); - protoInfo.SetState(TSourceIdInfo::ConvertState(memInfo.State)); - } - + auto& result = *resp.MutablePartitionResponse()->MutableCmdGetMaxSeqNoResult(); + for (const auto& sourceId : ev->Get()->SourceIds) { + auto& protoInfo = *result.AddSourceIdInfo(); + protoInfo.SetSourceId(sourceId); + + auto it = SourceIdStorage.GetInMemorySourceIds().find(sourceId); + if (it == SourceIdStorage.GetInMemorySourceIds().end()) { + continue; + } + + const auto& memInfo = it->second; + Y_VERIFY(memInfo.Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, memInfo.Offset); + Y_VERIFY(memInfo.SeqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, memInfo.SeqNo); + + protoInfo.SetSeqNo(memInfo.SeqNo); + protoInfo.SetOffset(memInfo.Offset); + protoInfo.SetWriteTimestampMS(memInfo.WriteTimestamp.MilliSeconds()); + protoInfo.SetExplicit(memInfo.Explicit); + protoInfo.SetState(TSourceIdInfo::ConvertState(memInfo.State)); + } + ctx.Send(Tablet, response.Release()); } @@ -2689,9 +2689,9 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) const ui64 quotedTime = Responses.front().QuotedTime; const ui64 queueTime = Responses.front().QueueTime; const ui64 writeTime = ctx.Now().MilliSeconds() - Responses.front().WriteTime; - - if (Responses.front().IsWrite()) { - const auto& writeResponse = Responses.front().GetWrite(); + + if (Responses.front().IsWrite()) { + const auto& writeResponse = Responses.front().GetWrite(); const TString& s = writeResponse.Msg.SourceId; const ui64& seqNo = writeResponse.Msg.SeqNo; const ui16& partNo = writeResponse.Msg.PartNo; @@ -2720,12 +2720,12 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) } } if (!already && partNo + 1 == totalParts) { - if (it == SourceIdStorage.GetInMemorySourceIds().end()) { + if (it == SourceIdStorage.GetInMemorySourceIds().end()) { Counters.Cumulative()[COUNTER_PQ_SID_CREATED].Increment(1); - SourceIdStorage.RegisterSourceId(s, writeResponse.Msg.SeqNo, offset, CurrentTimestamp); - } else { - SourceIdStorage.RegisterSourceId(s, it->second.Updated(writeResponse.Msg.SeqNo, offset, CurrentTimestamp)); - } + SourceIdStorage.RegisterSourceId(s, writeResponse.Msg.SeqNo, offset, CurrentTimestamp); + } else { + SourceIdStorage.RegisterSourceId(s, it->second.Updated(writeResponse.Msg.SeqNo, offset, CurrentTimestamp)); + } Counters.Cumulative()[COUNTER_PQ_WRITE_OK].Increment(1); } @@ -2746,50 +2746,50 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) if (!already && partNo + 1 == totalParts) ++offset; - } else if (Responses.front().IsOwnership()) { - const TString& ownerCookie = Responses.front().GetOwnership().OwnerCookie; + } else if (Responses.front().IsOwnership()) { + const TString& ownerCookie = Responses.front().GetOwnership().OwnerCookie; auto it = Owners.find(TOwnerInfo::GetOwnerFromOwnerCookie(ownerCookie)); if (it != Owners.end() && it->second.OwnerCookie == ownerCookie) { ReplyOwnerOk(ctx, Responses.front().GetCookie(), ownerCookie); } else { ReplyError(ctx, Responses.front().GetCookie(), NPersQueue::NErrorCode::WRONG_COOKIE, "new GetOwnership request is dropped already"); } - } else if (Responses.front().IsRegisterMessageGroup()) { - const auto& body = Responses.front().GetRegisterMessageGroup().Body; - - TMaybe<TPartitionKeyRange> keyRange; - if (body.KeyRange) { - keyRange = TPartitionKeyRange::Parse(*body.KeyRange); - } - - Y_VERIFY(body.AssignedOffset); - SourceIdStorage.RegisterSourceId(body.SourceId, body.SeqNo, *body.AssignedOffset, CurrentTimestamp, std::move(keyRange)); - ReplyOk(ctx, Responses.front().GetCookie()); - } else if (Responses.front().IsDeregisterMessageGroup()) { - const auto& body = Responses.front().GetDeregisterMessageGroup().Body; - - SourceIdStorage.DeregisterSourceId(body.SourceId); - ReplyOk(ctx, Responses.front().GetCookie()); - } else if (Responses.front().IsSplitMessageGroup()) { - const auto& split = Responses.front().GetSplitMessageGroup(); - - for (const auto& body : split.Deregistrations) { - SourceIdStorage.DeregisterSourceId(body.SourceId); - } - - for (const auto& body : split.Registrations) { - TMaybe<TPartitionKeyRange> keyRange; - if (body.KeyRange) { - keyRange = TPartitionKeyRange::Parse(*body.KeyRange); - } - - Y_VERIFY(body.AssignedOffset); - SourceIdStorage.RegisterSourceId(body.SourceId, body.SeqNo, *body.AssignedOffset, CurrentTimestamp, std::move(keyRange), true); - } - - ReplyOk(ctx, Responses.front().GetCookie()); - } else { - Y_FAIL("Unexpected message"); + } else if (Responses.front().IsRegisterMessageGroup()) { + const auto& body = Responses.front().GetRegisterMessageGroup().Body; + + TMaybe<TPartitionKeyRange> keyRange; + if (body.KeyRange) { + keyRange = TPartitionKeyRange::Parse(*body.KeyRange); + } + + Y_VERIFY(body.AssignedOffset); + SourceIdStorage.RegisterSourceId(body.SourceId, body.SeqNo, *body.AssignedOffset, CurrentTimestamp, std::move(keyRange)); + ReplyOk(ctx, Responses.front().GetCookie()); + } else if (Responses.front().IsDeregisterMessageGroup()) { + const auto& body = Responses.front().GetDeregisterMessageGroup().Body; + + SourceIdStorage.DeregisterSourceId(body.SourceId); + ReplyOk(ctx, Responses.front().GetCookie()); + } else if (Responses.front().IsSplitMessageGroup()) { + const auto& split = Responses.front().GetSplitMessageGroup(); + + for (const auto& body : split.Deregistrations) { + SourceIdStorage.DeregisterSourceId(body.SourceId); + } + + for (const auto& body : split.Registrations) { + TMaybe<TPartitionKeyRange> keyRange; + if (body.KeyRange) { + keyRange = TPartitionKeyRange::Parse(*body.KeyRange); + } + + Y_VERIFY(body.AssignedOffset); + SourceIdStorage.RegisterSourceId(body.SourceId, body.SeqNo, *body.AssignedOffset, CurrentTimestamp, std::move(keyRange), true); + } + + ReplyOk(ctx, Responses.front().GetCookie()); + } else { + Y_FAIL("Unexpected message"); } Responses.pop_front(); } @@ -3204,10 +3204,10 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx) } //Partition counters bool haveChanges = false; - if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionLabeledCounters.GetCounters()[METRIC_MAX_NUM_SIDS].Get()) { + if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionLabeledCounters.GetCounters()[METRIC_MAX_NUM_SIDS].Get()) { haveChanges = true; - PartitionLabeledCounters.GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); - PartitionLabeledCounters.GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); + PartitionLabeledCounters.GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); + PartitionLabeledCounters.GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); } TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now()); @@ -3656,7 +3656,7 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c for (auto& msg: ev->Get()->Msgs) { size += msg.Data.size(); bool needToChangeOffset = msg.PartNo + 1 == msg.TotalParts; - Requests.emplace_back(TWriteMsg{ev->Get()->Cookie, offset, std::move(msg)}, WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); + Requests.emplace_back(TWriteMsg{ev->Get()->Cookie, offset, std::move(msg)}, WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); if (offset && needToChangeOffset) ++*offset; } @@ -3668,100 +3668,100 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c UpdateWriteBufferIsFullState(ctx.Now()); } -void TPartition::HandleOnIdle(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx) { - HandleOnWrite(ev, ctx); - HandleWrites(ctx); -} - -void TPartition::HandleOnWrite(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx) { - const auto& body = ev->Get()->Body; - - auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); - if (it != SourceIdStorage.GetInMemorySourceIds().end()) { - if (!it->second.Explicit) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - "Trying to register implicitly registered SourceId"); - } - - switch (it->second.State) { - case TSourceIdInfo::EState::Registered: - return ReplyOk(ctx, ev->Get()->Cookie); - case TSourceIdInfo::EState::PendingRegistration: - if (!body.AfterSplit) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - "AfterSplit must be set"); - } - break; - default: - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::ERROR, - TStringBuilder() << "Unknown state: " << static_cast<ui32>(it->second.State)); - } - } else if (body.AfterSplit) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - "SourceId not found, registration cannot be completed"); - } - - WriteQuota.Update(ctx.Now()); - Requests.emplace_back(TRegisterMessageGroupMsg(*ev->Get()), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); -} - -void TPartition::HandleOnIdle(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx) { - HandleOnWrite(ev, ctx); - HandleWrites(ctx); -} - -void TPartition::HandleOnWrite(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx) { - const auto& body = ev->Get()->Body; - - auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); - if (it == SourceIdStorage.GetInMemorySourceIds().end()) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED, - "SourceId doesn't exist"); - } - - WriteQuota.Update(ctx.Now()); - Requests.emplace_back(TDeregisterMessageGroupMsg(*ev->Get()), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); -} - -void TPartition::HandleOnIdle(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx) { - HandleOnWrite(ev, ctx); - HandleWrites(ctx); -} - -void TPartition::HandleOnWrite(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx) { - if (ev->Get()->Deregistrations.size() > 1) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - TStringBuilder() << "Currently, single deregistrations are supported"); - } - - TSplitMessageGroupMsg msg(ev->Get()->Cookie); - - for (auto& body : ev->Get()->Deregistrations) { - auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); - if (it != SourceIdStorage.GetInMemorySourceIds().end()) { - msg.Deregistrations.push_back(std::move(body)); - } else { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED, - "SourceId doesn't exist"); - } - } - - for (auto& body : ev->Get()->Registrations) { - auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); - if (it == SourceIdStorage.GetInMemorySourceIds().end()) { - msg.Registrations.push_back(std::move(body)); - } else { - if (!it->second.Explicit) { - return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - "Trying to register implicitly registered SourceId"); - } - } - } - - WriteQuota.Update(ctx.Now()); - Requests.emplace_back(std::move(msg), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); -} - +void TPartition::HandleOnIdle(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx) { + HandleOnWrite(ev, ctx); + HandleWrites(ctx); +} + +void TPartition::HandleOnWrite(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx) { + const auto& body = ev->Get()->Body; + + auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); + if (it != SourceIdStorage.GetInMemorySourceIds().end()) { + if (!it->second.Explicit) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + "Trying to register implicitly registered SourceId"); + } + + switch (it->second.State) { + case TSourceIdInfo::EState::Registered: + return ReplyOk(ctx, ev->Get()->Cookie); + case TSourceIdInfo::EState::PendingRegistration: + if (!body.AfterSplit) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + "AfterSplit must be set"); + } + break; + default: + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::ERROR, + TStringBuilder() << "Unknown state: " << static_cast<ui32>(it->second.State)); + } + } else if (body.AfterSplit) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + "SourceId not found, registration cannot be completed"); + } + + WriteQuota.Update(ctx.Now()); + Requests.emplace_back(TRegisterMessageGroupMsg(*ev->Get()), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); +} + +void TPartition::HandleOnIdle(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx) { + HandleOnWrite(ev, ctx); + HandleWrites(ctx); +} + +void TPartition::HandleOnWrite(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx) { + const auto& body = ev->Get()->Body; + + auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); + if (it == SourceIdStorage.GetInMemorySourceIds().end()) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED, + "SourceId doesn't exist"); + } + + WriteQuota.Update(ctx.Now()); + Requests.emplace_back(TDeregisterMessageGroupMsg(*ev->Get()), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); +} + +void TPartition::HandleOnIdle(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx) { + HandleOnWrite(ev, ctx); + HandleWrites(ctx); +} + +void TPartition::HandleOnWrite(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx) { + if (ev->Get()->Deregistrations.size() > 1) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + TStringBuilder() << "Currently, single deregistrations are supported"); + } + + TSplitMessageGroupMsg msg(ev->Get()->Cookie); + + for (auto& body : ev->Get()->Deregistrations) { + auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); + if (it != SourceIdStorage.GetInMemorySourceIds().end()) { + msg.Deregistrations.push_back(std::move(body)); + } else { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED, + "SourceId doesn't exist"); + } + } + + for (auto& body : ev->Get()->Registrations) { + auto it = SourceIdStorage.GetInMemorySourceIds().find(body.SourceId); + if (it == SourceIdStorage.GetInMemorySourceIds().end()) { + msg.Registrations.push_back(std::move(body)); + } else { + if (!it->second.Explicit) { + return ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + "Trying to register implicitly registered SourceId"); + } + } + } + + WriteQuota.Update(ctx.Now()); + Requests.emplace_back(std::move(msg), WriteQuota.GetQuotedTime(), ctx.Now().MilliSeconds(), 0); +} + std::pair<TKey, ui32> TPartition::Compact(const TKey& key, const ui32 size, bool headCleared) { std::pair<TKey, ui32> res({key, size}); @@ -3944,7 +3944,7 @@ void TPartition::ClearOldHead(const ui64 offset, const ui16 partNo, TEvKeyValue: } -void TPartition::CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::TEvRequest* request, const TString& errorStr, const TWriteMsg& p, TSourceIdWriter& sourceIdWriter, NPersQueue::NErrorCode::EErrorCode errorCode = NPersQueue::NErrorCode::BAD_REQUEST) { +void TPartition::CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::TEvRequest* request, const TString& errorStr, const TWriteMsg& p, TSourceIdWriter& sourceIdWriter, NPersQueue::NErrorCode::EErrorCode errorCode = NPersQueue::NErrorCode::BAD_REQUEST) { ReplyError(ctx, p.Cookie, errorCode, errorStr); Counters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(p.Msg.Data.size() + p.Msg.SourceId.size()); @@ -3989,47 +3989,47 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const while (!Requests.empty() && WriteCycleSize < MAX_WRITE_CYCLE_SIZE) { //head is not too big auto pp = Requests.front(); Requests.pop_front(); - - if (!pp.IsWrite()) { - if (pp.IsRegisterMessageGroup()) { - auto& body = pp.GetRegisterMessageGroup().Body; - - TMaybe<TPartitionKeyRange> keyRange; - if (body.KeyRange) { - keyRange = TPartitionKeyRange::Parse(*body.KeyRange); - } - - body.AssignedOffset = curOffset; - sourceIdWriter.RegisterSourceId(body.SourceId, body.SeqNo, curOffset, CurrentTimestamp, std::move(keyRange)); - } else if (pp.IsDeregisterMessageGroup()) { - sourceIdWriter.DeregisterSourceId(pp.GetDeregisterMessageGroup().Body.SourceId); - } else if (pp.IsSplitMessageGroup()) { - for (auto& body : pp.GetSplitMessageGroup().Deregistrations) { - sourceIdWriter.DeregisterSourceId(body.SourceId); - } - - for (auto& body : pp.GetSplitMessageGroup().Registrations) { - TMaybe<TPartitionKeyRange> keyRange; - if (body.KeyRange) { - keyRange = TPartitionKeyRange::Parse(*body.KeyRange); - } - - body.AssignedOffset = curOffset; - sourceIdWriter.RegisterSourceId(body.SourceId, body.SeqNo, curOffset, CurrentTimestamp, std::move(keyRange), true); - } - } else { - Y_VERIFY(pp.IsOwnership()); - } - + + if (!pp.IsWrite()) { + if (pp.IsRegisterMessageGroup()) { + auto& body = pp.GetRegisterMessageGroup().Body; + + TMaybe<TPartitionKeyRange> keyRange; + if (body.KeyRange) { + keyRange = TPartitionKeyRange::Parse(*body.KeyRange); + } + + body.AssignedOffset = curOffset; + sourceIdWriter.RegisterSourceId(body.SourceId, body.SeqNo, curOffset, CurrentTimestamp, std::move(keyRange)); + } else if (pp.IsDeregisterMessageGroup()) { + sourceIdWriter.DeregisterSourceId(pp.GetDeregisterMessageGroup().Body.SourceId); + } else if (pp.IsSplitMessageGroup()) { + for (auto& body : pp.GetSplitMessageGroup().Deregistrations) { + sourceIdWriter.DeregisterSourceId(body.SourceId); + } + + for (auto& body : pp.GetSplitMessageGroup().Registrations) { + TMaybe<TPartitionKeyRange> keyRange; + if (body.KeyRange) { + keyRange = TPartitionKeyRange::Parse(*body.KeyRange); + } + + body.AssignedOffset = curOffset; + sourceIdWriter.RegisterSourceId(body.SourceId, body.SeqNo, curOffset, CurrentTimestamp, std::move(keyRange), true); + } + } else { + Y_VERIFY(pp.IsOwnership()); + } + pp.QuotedTime = WriteQuota.GetQuotedTime() - pp.QuotedTime; //change to duration pp.QueueTime = ctx.Now().MilliSeconds() - pp.QueueTime; pp.WriteTime = ctx.Now().MilliSeconds(); Responses.push_back(pp); continue; } - - Y_VERIFY(pp.IsWrite()); - auto& p = pp.GetWrite(); + + Y_VERIFY(pp.IsWrite()); + auto& p = pp.GetWrite(); WriteInflightSize -= p.Msg.Data.size(); @@ -4250,12 +4250,12 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const << " FormedBlobsCount " << PartitionedBlob.GetFormedBlobs().size() << " NewHead: " << NewHead); - if (it_inMemory == SourceIdStorage.GetInMemorySourceIds().end()) { - sourceIdWriter.RegisterSourceId(p.Msg.SourceId, p.Msg.SeqNo, curOffset, CurrentTimestamp); - } else { - sourceIdWriter.RegisterSourceId(p.Msg.SourceId, it_inMemory->second.Updated(p.Msg.SeqNo, curOffset, CurrentTimestamp)); - } - + if (it_inMemory == SourceIdStorage.GetInMemorySourceIds().end()) { + sourceIdWriter.RegisterSourceId(p.Msg.SourceId, p.Msg.SeqNo, curOffset, CurrentTimestamp); + } else { + sourceIdWriter.RegisterSourceId(p.Msg.SourceId, it_inMemory->second.Updated(p.Msg.SeqNo, curOffset, CurrentTimestamp)); + } + ++curOffset; PartitionedBlob = TPartitionedBlob(Partition, 0, "", 0, 0, 0, Head, NewHead, true, false, MaxBlobSize); } @@ -4468,10 +4468,10 @@ bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, const TActorCon Y_VERIFY(request->Record.CmdWriteSize() == 0); Y_VERIFY(request->Record.CmdRenameSize() == 0); Y_VERIFY(request->Record.CmdDeleteRangeSize() == 0); - const auto format = AppData(ctx)->PQConfig.GetEnableProtoSourceIdInfo() - ? ESourceIdFormat::Proto - : ESourceIdFormat::Raw; - TSourceIdWriter sourceIdWriter(format); + const auto format = AppData(ctx)->PQConfig.GetEnableProtoSourceIdInfo() + ? ESourceIdFormat::Proto + : ESourceIdFormat::Raw; + TSourceIdWriter sourceIdWriter(format); bool headCleared = AppendHeadWithNewWrites(request, ctx, sourceIdWriter); @@ -4487,7 +4487,7 @@ bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, const TActorCon return request->Record.CmdWriteSize() > 0 || request->Record.CmdRenameSize() > 0 || request->Record.CmdDeleteRangeSize() > 0; } - sourceIdWriter.FillRequest(request, Partition); + sourceIdWriter.FillRequest(request, Partition); std::pair<TKey, ui32> res = GetNewWriteKey(headCleared); const auto& key = res.first; @@ -4508,11 +4508,11 @@ void TPartition::FilterDeadlinedWrites(const TActorContext& ctx) for (auto& w : Requests) { ReplyError(ctx, w.GetCookie(), NPersQueue::NErrorCode::OVERLOAD, "quota exceeded"); - if (w.IsWrite()) { - const auto& msg = w.GetWrite().Msg; + if (w.IsWrite()) { + const auto& msg = w.GetWrite().Msg; Counters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); - Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); - WriteInflightSize -= msg.Data.size(); + Counters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); + WriteInflightSize -= msg.Data.size(); } } Requests.clear(); diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index 0c58929b0a1..2eec26f2516 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -20,8 +20,8 @@ #include "sourceid.h" #include "ownerinfo.h" -#include <variant> - +#include <variant> + namespace NKikimr { namespace NPQ { @@ -147,16 +147,16 @@ private: void HandleOnIdle(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ctx); void HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ctx); - - void HandleOnIdle(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx); - void HandleOnWrite(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx); - - void HandleOnIdle(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx); - void HandleOnWrite(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx); - - void HandleOnIdle(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx); - void HandleOnWrite(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx); - + + void HandleOnIdle(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx); + void HandleOnWrite(TEvPQ::TEvRegisterMessageGroup::TPtr& ev, const TActorContext& ctx); + + void HandleOnIdle(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx); + void HandleOnWrite(TEvPQ::TEvDeregisterMessageGroup::TPtr& ev, const TActorContext& ctx); + + void HandleOnIdle(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx); + void HandleOnWrite(TEvPQ::TEvSplitMessageGroup::TPtr& ev, const TActorContext& ctx); + void Handle(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); bool DropOldStuff(TEvKeyValue::TEvRequest* request, bool hasWrites, const TActorContext& ctx); @@ -187,9 +187,9 @@ private: //only Writes container is filled; only DISK_IS_FULL can be here void CancelAllWritesOnIdle(const TActorContext& ctx); //additional contaiters are half-filled, need to clear them too - struct TWriteMsg; // forward + struct TWriteMsg; // forward void CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::TEvRequest* request, const TString& errorStr, - const TWriteMsg& p, TSourceIdWriter& sourceIdWriter, NPersQueue::NErrorCode::EErrorCode errorCode); + const TWriteMsg& p, TSourceIdWriter& sourceIdWriter, NPersQueue::NErrorCode::EErrorCode errorCode); void FailBadClient(const TActorContext& ctx); @@ -313,9 +313,9 @@ private: HFuncTraced(TEvPQ::TEvPipeDisconnected, Handle); HFuncTraced(TEvQuota::TEvClearance, Handle); HFuncTraced(TEvPQ::TEvQuotaDeadlineCheck, Handle); - HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnIdle); - HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnIdle); - HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnIdle); + HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnIdle); + HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnIdle); + HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnIdle); default: LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Unexpected " << EventStr("StateIdle", ev)); @@ -361,10 +361,10 @@ private: HFuncTraced(TEvPQ::TEvUpdateAvailableSize, HandleOnWrite); HFuncTraced(TEvPQ::TEvQuotaDeadlineCheck, Handle); HFuncTraced(TEvQuota::TEvClearance, Handle); - HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnWrite); - HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnWrite); - HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnWrite); - + HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnWrite); + HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnWrite); + HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnWrite); + default: LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Unexpected " << EventStr("StateWrite", ev)); break; @@ -440,109 +440,109 @@ private: EInitState InitState; - struct TWriteMsg { - ui64 Cookie; - TMaybe<ui64> Offset; - TEvPQ::TEvWrite::TMsg Msg; - }; + struct TWriteMsg { + ui64 Cookie; + TMaybe<ui64> Offset; + TEvPQ::TEvWrite::TMsg Msg; + }; - struct TOwnershipMsg { + struct TOwnershipMsg { ui64 Cookie; TString OwnerCookie; }; - struct TRegisterMessageGroupMsg { - ui64 Cookie; - TEvPQ::TEvRegisterMessageGroup::TBody Body; - - explicit TRegisterMessageGroupMsg(TEvPQ::TEvRegisterMessageGroup& ev) - : Cookie(ev.Cookie) - , Body(std::move(ev.Body)) - { - } - }; - - struct TDeregisterMessageGroupMsg { - ui64 Cookie; - TEvPQ::TEvDeregisterMessageGroup::TBody Body; - - explicit TDeregisterMessageGroupMsg(TEvPQ::TEvDeregisterMessageGroup& ev) - : Cookie(ev.Cookie) - , Body(std::move(ev.Body)) - { - } - }; - - struct TSplitMessageGroupMsg { - ui64 Cookie; - TVector<TEvPQ::TEvDeregisterMessageGroup::TBody> Deregistrations; - TVector<TEvPQ::TEvRegisterMessageGroup::TBody> Registrations; - - explicit TSplitMessageGroupMsg(ui64 cookie) - : Cookie(cookie) - { - } - }; - + struct TRegisterMessageGroupMsg { + ui64 Cookie; + TEvPQ::TEvRegisterMessageGroup::TBody Body; + + explicit TRegisterMessageGroupMsg(TEvPQ::TEvRegisterMessageGroup& ev) + : Cookie(ev.Cookie) + , Body(std::move(ev.Body)) + { + } + }; + + struct TDeregisterMessageGroupMsg { + ui64 Cookie; + TEvPQ::TEvDeregisterMessageGroup::TBody Body; + + explicit TDeregisterMessageGroupMsg(TEvPQ::TEvDeregisterMessageGroup& ev) + : Cookie(ev.Cookie) + , Body(std::move(ev.Body)) + { + } + }; + + struct TSplitMessageGroupMsg { + ui64 Cookie; + TVector<TEvPQ::TEvDeregisterMessageGroup::TBody> Deregistrations; + TVector<TEvPQ::TEvRegisterMessageGroup::TBody> Registrations; + + explicit TSplitMessageGroupMsg(ui64 cookie) + : Cookie(cookie) + { + } + }; + struct TMessage { - std::variant< - TWriteMsg, - TOwnershipMsg, - TRegisterMessageGroupMsg, - TDeregisterMessageGroupMsg, - TSplitMessageGroupMsg - > Body; + std::variant< + TWriteMsg, + TOwnershipMsg, + TRegisterMessageGroupMsg, + TDeregisterMessageGroupMsg, + TSplitMessageGroupMsg + > Body; ui64 QuotedTime; ui64 QueueTime; ui64 WriteTime; - template <typename T> - explicit TMessage(T&& body, ui64 quotedTime, ui64 queueTime, ui64 writeTime) - : Body(std::forward<T>(body)) - , QuotedTime(quotedTime) - , QueueTime(queueTime) - , WriteTime(writeTime) - { - } - + template <typename T> + explicit TMessage(T&& body, ui64 quotedTime, ui64 queueTime, ui64 writeTime) + : Body(std::forward<T>(body)) + , QuotedTime(quotedTime) + , QueueTime(queueTime) + , WriteTime(writeTime) + { + } + ui64 GetCookie() const { - switch (Body.index()) { - case 0: - return std::get<0>(Body).Cookie; - case 1: - return std::get<1>(Body).Cookie; - case 2: - return std::get<2>(Body).Cookie; - case 3: - return std::get<3>(Body).Cookie; - case 4: - return std::get<4>(Body).Cookie; - default: - Y_FAIL("unreachable"); - } + switch (Body.index()) { + case 0: + return std::get<0>(Body).Cookie; + case 1: + return std::get<1>(Body).Cookie; + case 2: + return std::get<2>(Body).Cookie; + case 3: + return std::get<3>(Body).Cookie; + case 4: + return std::get<4>(Body).Cookie; + default: + Y_FAIL("unreachable"); + } } - - #define DEFINE_CHECKER_GETTER(name, i) \ - bool Is##name() const { \ - return Body.index() == i; \ - } \ - const auto& Get##name() const { \ - Y_VERIFY(Is##name()); \ - return std::get<i>(Body); \ - } \ - auto& Get##name() { \ - Y_VERIFY(Is##name()); \ - return std::get<i>(Body); \ - } - - DEFINE_CHECKER_GETTER(Write, 0) - DEFINE_CHECKER_GETTER(Ownership, 1) - DEFINE_CHECKER_GETTER(RegisterMessageGroup, 2) - DEFINE_CHECKER_GETTER(DeregisterMessageGroup, 3) - DEFINE_CHECKER_GETTER(SplitMessageGroup, 4) - - #undef DEFINE_CHECKER_GETTER + + #define DEFINE_CHECKER_GETTER(name, i) \ + bool Is##name() const { \ + return Body.index() == i; \ + } \ + const auto& Get##name() const { \ + Y_VERIFY(Is##name()); \ + return std::get<i>(Body); \ + } \ + auto& Get##name() { \ + Y_VERIFY(Is##name()); \ + return std::get<i>(Body); \ + } + + DEFINE_CHECKER_GETTER(Write, 0) + DEFINE_CHECKER_GETTER(Ownership, 1) + DEFINE_CHECKER_GETTER(RegisterMessageGroup, 2) + DEFINE_CHECKER_GETTER(DeregisterMessageGroup, 3) + DEFINE_CHECKER_GETTER(SplitMessageGroup, 4) + + #undef DEFINE_CHECKER_GETTER }; std::deque<TMessage> Requests; diff --git a/ydb/core/persqueue/partition_key_range/partition_key_range.cpp b/ydb/core/persqueue/partition_key_range/partition_key_range.cpp index ff0c049855d..3c1a2b36af8 100644 --- a/ydb/core/persqueue/partition_key_range/partition_key_range.cpp +++ b/ydb/core/persqueue/partition_key_range/partition_key_range.cpp @@ -1,39 +1,39 @@ -#include "partition_key_range.h" - +#include "partition_key_range.h" + #include <ydb/core/protos/pqconfig.pb.h> - -namespace NKikimr { -namespace NPQ { - -TPartitionKeyRange TPartitionKeyRange::Parse(const NKikimrPQ::TPartitionKeyRange& proto) { - TPartitionKeyRange result; - - if (proto.HasFromBound()) { - ParseBound(proto.GetFromBound(), result.FromBound); - } - - if (proto.HasToBound()) { - ParseBound(proto.GetToBound(), result.ToBound); - } - - return result; -} - -void TPartitionKeyRange::Serialize(NKikimrPQ::TPartitionKeyRange& proto) const { - if (FromBound) { - proto.SetFromBound(FromBound->GetBuffer()); - } - - if (ToBound) { - proto.SetToBound(ToBound->GetBuffer()); - } -} - -void TPartitionKeyRange::ParseBound(const TString& data, TMaybe<TSerializedCellVec>& bound) { - TSerializedCellVec cells; - Y_VERIFY(TSerializedCellVec::TryParse(data, cells)); - bound.ConstructInPlace(std::move(cells)); -} - -} // NPQ -} // NKikimr + +namespace NKikimr { +namespace NPQ { + +TPartitionKeyRange TPartitionKeyRange::Parse(const NKikimrPQ::TPartitionKeyRange& proto) { + TPartitionKeyRange result; + + if (proto.HasFromBound()) { + ParseBound(proto.GetFromBound(), result.FromBound); + } + + if (proto.HasToBound()) { + ParseBound(proto.GetToBound(), result.ToBound); + } + + return result; +} + +void TPartitionKeyRange::Serialize(NKikimrPQ::TPartitionKeyRange& proto) const { + if (FromBound) { + proto.SetFromBound(FromBound->GetBuffer()); + } + + if (ToBound) { + proto.SetToBound(ToBound->GetBuffer()); + } +} + +void TPartitionKeyRange::ParseBound(const TString& data, TMaybe<TSerializedCellVec>& bound) { + TSerializedCellVec cells; + Y_VERIFY(TSerializedCellVec::TryParse(data, cells)); + bound.ConstructInPlace(std::move(cells)); +} + +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/partition_key_range/partition_key_range.h b/ydb/core/persqueue/partition_key_range/partition_key_range.h index d3aac74a07e..c209e57a869 100644 --- a/ydb/core/persqueue/partition_key_range/partition_key_range.h +++ b/ydb/core/persqueue/partition_key_range/partition_key_range.h @@ -1,28 +1,28 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_tablecell.h> - -#include <util/generic/maybe.h> - -// forward declarations -namespace NKikimrPQ { - class TPartitionKeyRange; -} - -namespace NKikimr { -namespace NPQ { - -struct TPartitionKeyRange { - TMaybe<TSerializedCellVec> FromBound; // inclusive - TMaybe<TSerializedCellVec> ToBound; // exclusive - - static TPartitionKeyRange Parse(const NKikimrPQ::TPartitionKeyRange& proto); - void Serialize(NKikimrPQ::TPartitionKeyRange& proto) const; - -private: - static void ParseBound(const TString& data, TMaybe<TSerializedCellVec>& bound); - -}; // TPartitionKeyRange - -} // NPQ -} // NKikimr + +#include <util/generic/maybe.h> + +// forward declarations +namespace NKikimrPQ { + class TPartitionKeyRange; +} + +namespace NKikimr { +namespace NPQ { + +struct TPartitionKeyRange { + TMaybe<TSerializedCellVec> FromBound; // inclusive + TMaybe<TSerializedCellVec> ToBound; // exclusive + + static TPartitionKeyRange Parse(const NKikimrPQ::TPartitionKeyRange& proto); + void Serialize(NKikimrPQ::TPartitionKeyRange& proto) const; + +private: + static void ParseBound(const TString& data, TMaybe<TSerializedCellVec>& bound); + +}; // TPartitionKeyRange + +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/partition_key_range/ya.make b/ydb/core/persqueue/partition_key_range/ya.make index c4d153a9d0f..d67d12e3edf 100644 --- a/ydb/core/persqueue/partition_key_range/ya.make +++ b/ydb/core/persqueue/partition_key_range/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - ilnaz +LIBRARY() + +OWNER( + ilnaz g:kikimr - g:logbroker -) - -SRCS( - partition_key_range.cpp -) - -PEERDIR( + g:logbroker +) + +SRCS( + partition_key_range.cpp +) + +PEERDIR( ydb/core/protos ydb/core/scheme -) - -END() +) + +END() diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index d66dad3861c..9937dbe561c 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -26,13 +26,13 @@ const TString TMP_REQUEST_MARKER = "__TMP__REQUEST__MARKER__"; const ui32 CACHE_SIZE = 100 << 20; //100mb per tablet by default const ui32 MAX_BYTES = 25 * 1024 * 1024; const TDuration TOTAL_TIMEOUT = TDuration::Seconds(120); -static constexpr ui32 MAX_SOURCE_ID_LENGTH = 10240; +static constexpr ui32 MAX_SOURCE_ID_LENGTH = 10240; struct TPartitionInfo { - TPartitionInfo(const TActorId& actor, TMaybe<TPartitionKeyRange>&& keyRange, - const bool initDone, const TTabletCountersBase& baseline) + TPartitionInfo(const TActorId& actor, TMaybe<TPartitionKeyRange>&& keyRange, + const bool initDone, const TTabletCountersBase& baseline) : Actor(actor) - , KeyRange(std::move(keyRange)) + , KeyRange(std::move(keyRange)) , InitDone(initDone) { Baseline.Populate(baseline); @@ -40,14 +40,14 @@ struct TPartitionInfo { TPartitionInfo(const TPartitionInfo& info) : Actor(info.Actor) - , KeyRange(info.KeyRange) + , KeyRange(info.KeyRange) , InitDone(info.InitDone) { Baseline.Populate(info.Baseline); } TActorId Actor; - TMaybe<TPartitionKeyRange> KeyRange; + TMaybe<TPartitionKeyRange> KeyRange; bool InitDone; TTabletCountersBase Baseline; THashMap<TString, TTabletLabeledCountersBase> LabeledCounters; @@ -75,12 +75,12 @@ struct TChangeNotification { ui64 TxId; }; -static TMaybe<TPartitionKeyRange> GetPartitionKeyRange(const NKikimrPQ::TPQTabletConfig::TPartition& proto) { - if (!proto.HasKeyRange()) { - return Nothing(); - } - return TPartitionKeyRange::Parse(proto.GetKeyRange()); -} +static TMaybe<TPartitionKeyRange> GetPartitionKeyRange(const NKikimrPQ::TPQTabletConfig::TPartition& proto) { + if (!proto.HasKeyRange()) { + return Nothing(); + } + return TPartitionKeyRange::Parse(proto.GetKeyRange()); +} /******************************************************* ReadProxy *********************************************************/ //megaqc - remove it when LB will be ready @@ -563,17 +563,17 @@ void TPersQueue::FillMeteringParams(const TActorContext& ctx) void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) { THashSet<ui32> was; - if (NewConfig.PartitionsSize()) { - for (const auto& partition : NewConfig.GetPartitions()) { - was.insert(partition.GetPartitionId()); - } - } else { - for (const auto partitionId : NewConfig.GetPartitionIds()) { - was.insert(partitionId); - } + if (NewConfig.PartitionsSize()) { + for (const auto& partition : NewConfig.GetPartitions()) { + was.insert(partition.GetPartitionId()); + } + } else { + for (const auto partitionId : NewConfig.GetPartitionIds()) { + was.insert(partitionId); + } } - for (const auto& partition : Config.GetPartitions()) { - Y_VERIFY_S(was.contains(partition.GetPartitionId()), "New config is bad, missing partition " << partition.GetPartitionId()); + for (const auto& partition : Config.GetPartitions()) { + Y_VERIFY_S(was.contains(partition.GetPartitionId()), "New config is bad, missing partition " << partition.GetPartitionId()); } Y_VERIFY(ConfigInited && PartitionsInited == Partitions.size()); //in order to answer only after all parts are ready to work @@ -584,12 +584,12 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) FillMeteringParams(ctx); - if (!Config.PartitionsSize()) { - for (const auto partitionId : Config.GetPartitionIds()) { - Config.AddPartitions()->SetPartitionId(partitionId); - } - } - + if (!Config.PartitionsSize()) { + for (const auto partitionId : Config.GetPartitionIds()) { + Config.AddPartitions()->SetPartitionId(partitionId); + } + } + ui32 cacheSize = CACHE_SIZE; if (Config.HasCacheSize()) cacheSize = Config.GetCacheSize(); @@ -598,13 +598,13 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) TopicName = Config.GetTopicName(); TopicPath = Config.GetTopicPath(); LocalDC = Config.GetLocalDC(); - - KeySchema.clear(); - KeySchema.reserve(Config.PartitionKeySchemaSize()); - for (const auto& component : Config.GetPartitionKeySchema()) { - KeySchema.push_back(component.GetTypeId()); - } - + + KeySchema.clear(); + KeySchema.reserve(Config.PartitionKeySchemaSize()); + for (const auto& component : Config.GetPartitionKeySchema()) { + KeySchema.push_back(component.GetTypeId()); + } + Y_VERIFY(TopicName.size(), "Need topic name here"); CacheActor = ctx.Register(new TPQCacheProxy(ctx.SelfID, TopicName, cacheSize)); } else { @@ -615,17 +615,17 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) for (auto& p : Partitions) { //change config for already created partitions ctx.Send(p.second.Actor, new TEvPQ::TEvChangeConfig(TopicName, Config)); } - for (const auto& partition : Config.GetPartitions()) { - const auto partitionId = partition.GetPartitionId(); - if (Partitions.find(partitionId) == Partitions.end()) { - Partitions.emplace(partitionId, TPartitionInfo( - ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, true)), - GetPartitionKeyRange(partition), + for (const auto& partition : Config.GetPartitions()) { + const auto partitionId = partition.GetPartitionId(); + if (Partitions.find(partitionId) == Partitions.end()) { + Partitions.emplace(partitionId, TPartitionInfo( + ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, true)), + GetPartitionKeyRange(partition), true, *Counters - )); - - // InitCompleted is true because this partition is empty + )); + + // InitCompleted is true because this partition is empty ++PartitionsInited; //newly created partition is empty and ready to work } } @@ -696,22 +696,22 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& if (read.GetStatus() == NKikimrProto::OK) { bool res = Config.ParseFromString(read.GetValue()); Y_VERIFY(res); - - if (!Config.PartitionsSize()) { - for (const auto partitionId : Config.GetPartitionIds()) { - Config.AddPartitions()->SetPartitionId(partitionId); - } - } - + + if (!Config.PartitionsSize()) { + for (const auto partitionId : Config.GetPartitionIds()) { + Config.AddPartitions()->SetPartitionId(partitionId); + } + } + TopicName = Config.GetTopicName(); LocalDC = Config.GetLocalDC(); - KeySchema.clear(); - KeySchema.reserve(Config.PartitionKeySchemaSize()); - for (const auto& component : Config.GetPartitionKeySchema()) { - KeySchema.push_back(component.GetTypeId()); - } - + KeySchema.clear(); + KeySchema.reserve(Config.PartitionKeySchemaSize()); + for (const auto& component : Config.GetPartitionKeySchema()) { + KeySchema.push_back(component.GetTypeId()); + } + ui32 cacheSize = CACHE_SIZE; if (Config.HasCacheSize()) cacheSize = Config.GetCacheSize(); @@ -724,14 +724,14 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& Y_FAIL("Unexpected config read status: %d", read.GetStatus()); } - for (const auto& partition : Config.GetPartitions()) { // no partitions will be created with empty config - const auto partitionId = partition.GetPartitionId(); - Partitions.emplace(partitionId, TPartitionInfo( - ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, false)), - GetPartitionKeyRange(partition), + for (const auto& partition : Config.GetPartitions()) { // no partitions will be created with empty config + const auto partitionId = partition.GetPartitionId(); + Partitions.emplace(partitionId, TPartitionInfo( + ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, false)), + GetPartitionKeyRange(partition), false, *Counters - )); + )); } ConfigInited = true; @@ -1059,48 +1059,48 @@ void TPersQueue::ProcessUpdateConfigRequest(TAutoPtr<TEvPersQueue::TEvUpdateConf return; } - const auto& bootstrapCfg = record.GetBootstrapConfig(); - - if (bootstrapCfg.ExplicitMessageGroupsSize() && !AppData(ctx)->PQConfig.GetEnableProtoSourceIdInfo()) { - LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() - << ": cannot apply explicit message groups unless proto source id enabled" - << ", actor " << sender - << ", txId " << record.GetTxId()); - - THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; - res->Record.SetStatus(NKikimrPQ::ERROR); - res->Record.SetTxId(record.GetTxId()); - res->Record.SetOrigin(TabletID()); - ctx.Send(sender, res.Release()); - return; - } - - for (const auto& mg : bootstrapCfg.GetExplicitMessageGroups()) { - TString error; - - if (!mg.HasId() || mg.GetId().empty()) { - error = "Empty Id"; - } else if (mg.GetId().size() > MAX_SOURCE_ID_LENGTH) { - error = "Too long Id"; - } else if (mg.HasKeyRange() && !cfg.PartitionKeySchemaSize()) { - error = "Missing KeySchema"; - } - - if (error) { - LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() - << "Cannot apply explicit message group: " << error - << " actor " << sender - << " txId " << record.GetTxId()); - - THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; - res->Record.SetStatus(NKikimrPQ::ERROR); - res->Record.SetTxId(record.GetTxId()); - res->Record.SetOrigin(TabletID()); - ctx.Send(sender, res.Release()); - return; - } - } - + const auto& bootstrapCfg = record.GetBootstrapConfig(); + + if (bootstrapCfg.ExplicitMessageGroupsSize() && !AppData(ctx)->PQConfig.GetEnableProtoSourceIdInfo()) { + LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() + << ": cannot apply explicit message groups unless proto source id enabled" + << ", actor " << sender + << ", txId " << record.GetTxId()); + + THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; + res->Record.SetStatus(NKikimrPQ::ERROR); + res->Record.SetTxId(record.GetTxId()); + res->Record.SetOrigin(TabletID()); + ctx.Send(sender, res.Release()); + return; + } + + for (const auto& mg : bootstrapCfg.GetExplicitMessageGroups()) { + TString error; + + if (!mg.HasId() || mg.GetId().empty()) { + error = "Empty Id"; + } else if (mg.GetId().size() > MAX_SOURCE_ID_LENGTH) { + error = "Too long Id"; + } else if (mg.HasKeyRange() && !cfg.PartitionKeySchemaSize()) { + error = "Missing KeySchema"; + } + + if (error) { + LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() + << "Cannot apply explicit message group: " << error + << " actor " << sender + << " txId " << record.GetTxId()); + + THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; + res->Record.SetStatus(NKikimrPQ::ERROR); + res->Record.SetTxId(record.GetTxId()); + res->Record.SetOrigin(TabletID()); + ctx.Send(sender, res.Release()); + return; + } + } + ChangeConfigNotification.insert(TChangeNotification(sender, record.GetTxId())); if (!cfg.HasPartitionConfig()) @@ -1139,29 +1139,29 @@ void TPersQueue::ProcessUpdateConfigRequest(TAutoPtr<TEvPersQueue::TEvUpdateConf bool res = cfg.SerializeToString(&str); Y_VERIFY(res); - + TAutoPtr<TEvKeyValue::TEvRequest> request(new TEvKeyValue::TEvRequest); request->Record.SetCookie(WRITE_CONFIG_COOKIE); - + auto write = request->Record.AddCmdWrite(); write->SetKey(KeyConfig()); write->SetValue(str); write->SetTactic(AppData(ctx)->PQConfig.GetTactic()); - - TSourceIdWriter sourceIdWriter(ESourceIdFormat::Proto); - for (const auto& mg : bootstrapCfg.GetExplicitMessageGroups()) { - TMaybe<TPartitionKeyRange> keyRange; - if (mg.HasKeyRange()) { - keyRange = TPartitionKeyRange::Parse(mg.GetKeyRange()); - } - - sourceIdWriter.RegisterSourceId(mg.GetId(), 0, 0, ctx.Now(), std::move(keyRange)); - - for (const auto& partition : cfg.GetPartitions()) { - sourceIdWriter.FillRequest(request.Get(), partition.GetPartitionId()); - } - } - + + TSourceIdWriter sourceIdWriter(ESourceIdFormat::Proto); + for (const auto& mg : bootstrapCfg.GetExplicitMessageGroups()) { + TMaybe<TPartitionKeyRange> keyRange; + if (mg.HasKeyRange()) { + keyRange = TPartitionKeyRange::Parse(mg.GetKeyRange()); + } + + sourceIdWriter.RegisterSourceId(mg.GetId(), 0, 0, ctx.Now(), std::move(keyRange)); + + for (const auto& partition : cfg.GetPartitions()) { + sourceIdWriter.FillRequest(request.Get(), partition.GetPartitionId()); + } + } + NewConfig = cfg; ctx.Send(ctx.SelfID, request.Release()); } @@ -1477,7 +1477,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p errorStr = "TotalSize must be filled for first part"; } else if (cmd.HasTotalSize() && static_cast<size_t>(cmd.GetTotalSize()) <= cmd.GetData().size()) { // TotalSize must be > size of each part errorStr = "TotalSize is incorrect"; - } else if (cmd.GetSourceId().size() > MAX_SOURCE_ID_LENGTH) { + } else if (cmd.GetSourceId().size() > MAX_SOURCE_ID_LENGTH) { errorStr = "Too big SourceId"; } else if (mirroredPartition && !cmd.GetDisableDeduplication()) { errorStr = "Write to mirrored topic is forbiden"; @@ -1673,120 +1673,120 @@ void TPersQueue::HandleReadRequest(const ui64 responseCookie, const TActorId& pa } } -TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> TPersQueue::MakeRegisterMessageGroup( - const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd, - NPersQueue::NErrorCode::EErrorCode& code, TString& error) const -{ - if (!cmd.HasId() || cmd.GetId().empty()) { - code = NPersQueue::NErrorCode::BAD_REQUEST; - error = "Empty Id"; - return Nothing(); - } else if (cmd.GetId().size() > MAX_SOURCE_ID_LENGTH) { - code = NPersQueue::NErrorCode::BAD_REQUEST; - error = "Too long Id"; - return Nothing(); - } else if (cmd.HasPartitionKeyRange() && !Config.PartitionKeySchemaSize()) { - code = NPersQueue::NErrorCode::BAD_REQUEST; - error = "Missing KeySchema"; - return Nothing(); - } - - TMaybe<NKikimrPQ::TPartitionKeyRange> keyRange; - if (cmd.HasPartitionKeyRange()) { - keyRange = cmd.GetPartitionKeyRange(); - } - - return TEvPQ::TEvRegisterMessageGroup::TBody(cmd.GetId(), std::move(keyRange), cmd.GetStartingSeqNo(), cmd.GetAfterSplit()); -} - -TMaybe<TEvPQ::TEvDeregisterMessageGroup::TBody> TPersQueue::MakeDeregisterMessageGroup( - const NKikimrClient::TPersQueuePartitionRequest::TCmdDeregisterMessageGroup& cmd, - NPersQueue::NErrorCode::EErrorCode& code, TString& error) const -{ - if (!cmd.HasId() || cmd.GetId().empty()) { - code = NPersQueue::NErrorCode::BAD_REQUEST; - error = "Empty Id"; - return Nothing(); - } else if (cmd.GetId().size() > MAX_SOURCE_ID_LENGTH) { - code = NPersQueue::NErrorCode::BAD_REQUEST; - error = "Too long Id"; - return Nothing(); - } - - return TEvPQ::TEvDeregisterMessageGroup::TBody(cmd.GetId()); -} - -void TPersQueue::HandleRegisterMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, - const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) -{ - Y_VERIFY(req.HasCmdRegisterMessageGroup()); - - NPersQueue::NErrorCode::EErrorCode code; - TString error; - auto body = MakeRegisterMessageGroup(req.GetCmdRegisterMessageGroup(), code, error); - - if (!body) { - return ReplyError(ctx, responseCookie, code, error); - } - - InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_REGISTER_MESSAGE_GROUP); - ctx.Send(partActor, new TEvPQ::TEvRegisterMessageGroup(responseCookie, std::move(body.GetRef()))); -} - -void TPersQueue::HandleDeregisterMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, - const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) -{ - Y_VERIFY(req.HasCmdDeregisterMessageGroup()); - - NPersQueue::NErrorCode::EErrorCode code; - TString error; - auto body = MakeDeregisterMessageGroup(req.GetCmdDeregisterMessageGroup(), code, error); - - if (!body) { - return ReplyError(ctx, responseCookie, code, error); - } - - InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_DEREGISTER_MESSAGE_GROUP); - ctx.Send(partActor, new TEvPQ::TEvDeregisterMessageGroup(responseCookie, std::move(body.GetRef()))); -} - -void TPersQueue::HandleSplitMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, - const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) -{ - Y_VERIFY(req.HasCmdSplitMessageGroup()); - const auto& cmd = req.GetCmdSplitMessageGroup(); - - NPersQueue::NErrorCode::EErrorCode code; - TString error; - - TVector<TEvPQ::TEvDeregisterMessageGroup::TBody> deregistrations; - for (const auto& group : cmd.GetDeregisterGroups()) { - auto body = MakeDeregisterMessageGroup(group, code, error); - if (!body) { - return ReplyError(ctx, responseCookie, code, error); - } - - deregistrations.push_back(std::move(body.GetRef())); - } - - TVector<TEvPQ::TEvRegisterMessageGroup::TBody> registrations; - for (const auto& group : cmd.GetRegisterGroups()) { - if (group.GetAfterSplit()) { - return ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "AfterSplit cannot be set"); - } - - auto body = MakeRegisterMessageGroup(group, code, error); - if (!body) { - return ReplyError(ctx, responseCookie, code, error); - } - - registrations.push_back(std::move(body.GetRef())); - } - - InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_SPLIT_MESSAGE_GROUP); - ctx.Send(partActor, new TEvPQ::TEvSplitMessageGroup(responseCookie, std::move(deregistrations), std::move(registrations))); -} - +TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> TPersQueue::MakeRegisterMessageGroup( + const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd, + NPersQueue::NErrorCode::EErrorCode& code, TString& error) const +{ + if (!cmd.HasId() || cmd.GetId().empty()) { + code = NPersQueue::NErrorCode::BAD_REQUEST; + error = "Empty Id"; + return Nothing(); + } else if (cmd.GetId().size() > MAX_SOURCE_ID_LENGTH) { + code = NPersQueue::NErrorCode::BAD_REQUEST; + error = "Too long Id"; + return Nothing(); + } else if (cmd.HasPartitionKeyRange() && !Config.PartitionKeySchemaSize()) { + code = NPersQueue::NErrorCode::BAD_REQUEST; + error = "Missing KeySchema"; + return Nothing(); + } + + TMaybe<NKikimrPQ::TPartitionKeyRange> keyRange; + if (cmd.HasPartitionKeyRange()) { + keyRange = cmd.GetPartitionKeyRange(); + } + + return TEvPQ::TEvRegisterMessageGroup::TBody(cmd.GetId(), std::move(keyRange), cmd.GetStartingSeqNo(), cmd.GetAfterSplit()); +} + +TMaybe<TEvPQ::TEvDeregisterMessageGroup::TBody> TPersQueue::MakeDeregisterMessageGroup( + const NKikimrClient::TPersQueuePartitionRequest::TCmdDeregisterMessageGroup& cmd, + NPersQueue::NErrorCode::EErrorCode& code, TString& error) const +{ + if (!cmd.HasId() || cmd.GetId().empty()) { + code = NPersQueue::NErrorCode::BAD_REQUEST; + error = "Empty Id"; + return Nothing(); + } else if (cmd.GetId().size() > MAX_SOURCE_ID_LENGTH) { + code = NPersQueue::NErrorCode::BAD_REQUEST; + error = "Too long Id"; + return Nothing(); + } + + return TEvPQ::TEvDeregisterMessageGroup::TBody(cmd.GetId()); +} + +void TPersQueue::HandleRegisterMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, + const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) +{ + Y_VERIFY(req.HasCmdRegisterMessageGroup()); + + NPersQueue::NErrorCode::EErrorCode code; + TString error; + auto body = MakeRegisterMessageGroup(req.GetCmdRegisterMessageGroup(), code, error); + + if (!body) { + return ReplyError(ctx, responseCookie, code, error); + } + + InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_REGISTER_MESSAGE_GROUP); + ctx.Send(partActor, new TEvPQ::TEvRegisterMessageGroup(responseCookie, std::move(body.GetRef()))); +} + +void TPersQueue::HandleDeregisterMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, + const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) +{ + Y_VERIFY(req.HasCmdDeregisterMessageGroup()); + + NPersQueue::NErrorCode::EErrorCode code; + TString error; + auto body = MakeDeregisterMessageGroup(req.GetCmdDeregisterMessageGroup(), code, error); + + if (!body) { + return ReplyError(ctx, responseCookie, code, error); + } + + InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_DEREGISTER_MESSAGE_GROUP); + ctx.Send(partActor, new TEvPQ::TEvDeregisterMessageGroup(responseCookie, std::move(body.GetRef()))); +} + +void TPersQueue::HandleSplitMessageGroupRequest(ui64 responseCookie, const TActorId& partActor, + const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) +{ + Y_VERIFY(req.HasCmdSplitMessageGroup()); + const auto& cmd = req.GetCmdSplitMessageGroup(); + + NPersQueue::NErrorCode::EErrorCode code; + TString error; + + TVector<TEvPQ::TEvDeregisterMessageGroup::TBody> deregistrations; + for (const auto& group : cmd.GetDeregisterGroups()) { + auto body = MakeDeregisterMessageGroup(group, code, error); + if (!body) { + return ReplyError(ctx, responseCookie, code, error); + } + + deregistrations.push_back(std::move(body.GetRef())); + } + + TVector<TEvPQ::TEvRegisterMessageGroup::TBody> registrations; + for (const auto& group : cmd.GetRegisterGroups()) { + if (group.GetAfterSplit()) { + return ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "AfterSplit cannot be set"); + } + + auto body = MakeRegisterMessageGroup(group, code, error); + if (!body) { + return ReplyError(ctx, responseCookie, code, error); + } + + registrations.push_back(std::move(body.GetRef())); + } + + InitResponseBuilder(responseCookie, 1, COUNTER_LATENCY_PQ_SPLIT_MESSAGE_GROUP); + ctx.Send(partActor, new TEvPQ::TEvSplitMessageGroup(responseCookie, std::move(deregistrations), std::move(registrations))); +} + void TPersQueue::Handle(TEvPersQueue::TEvRequest::TPtr& ev, const TActorContext& ctx) { NKikimrClient::TPersQueueRequest& request = ev->Get()->Record; @@ -1845,19 +1845,19 @@ void TPersQueue::Handle(TEvPersQueue::TEvRequest::TPtr& ev, const TActorContext& return; } - ui32 count = req.HasCmdGetMaxSeqNo() - + req.HasCmdDeleteSession() - + req.HasCmdCreateSession() - + req.HasCmdSetClientOffset() - + req.HasCmdGetClientOffset() - + req.HasCmdRead() - + req.HasCmdGetOwnership() - + (req.CmdWriteSize() > 0 ? 1 : 0) - + req.HasCmdReserveBytes() - + req.HasCmdUpdateWriteTimestamp() - + req.HasCmdRegisterMessageGroup() - + req.HasCmdDeregisterMessageGroup() - + req.HasCmdSplitMessageGroup(); + ui32 count = req.HasCmdGetMaxSeqNo() + + req.HasCmdDeleteSession() + + req.HasCmdCreateSession() + + req.HasCmdSetClientOffset() + + req.HasCmdGetClientOffset() + + req.HasCmdRead() + + req.HasCmdGetOwnership() + + (req.CmdWriteSize() > 0 ? 1 : 0) + + req.HasCmdReserveBytes() + + req.HasCmdUpdateWriteTimestamp() + + req.HasCmdRegisterMessageGroup() + + req.HasCmdDeregisterMessageGroup() + + req.HasCmdSplitMessageGroup(); if (count != 1) { ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, @@ -1889,12 +1889,12 @@ void TPersQueue::Handle(TEvPersQueue::TEvRequest::TPtr& ev, const TActorContext& HandleGetOwnershipRequest(responseCookie, partActor, req, ctx, pipeClient, ev->Sender); } else if (req.HasCmdReserveBytes()) { HandleReserveBytesRequest(responseCookie, partActor, req, ctx, pipeClient, ev->Sender); - } else if (req.HasCmdRegisterMessageGroup()) { - HandleRegisterMessageGroupRequest(responseCookie, partActor, req, ctx); - } else if (req.HasCmdDeregisterMessageGroup()) { - HandleDeregisterMessageGroupRequest(responseCookie, partActor, req, ctx); - } else if (req.HasCmdSplitMessageGroup()) { - HandleSplitMessageGroupRequest(responseCookie, partActor, req, ctx); + } else if (req.HasCmdRegisterMessageGroup()) { + HandleRegisterMessageGroupRequest(responseCookie, partActor, req, ctx); + } else if (req.HasCmdDeregisterMessageGroup()) { + HandleDeregisterMessageGroupRequest(responseCookie, partActor, req, ctx); + } else if (req.HasCmdSplitMessageGroup()) { + HandleSplitMessageGroupRequest(responseCookie, partActor, req, ctx); } else Y_FAIL("unknown or empty command"); } diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index aa161552b3c..bc3bfb10ba5 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -88,14 +88,14 @@ class TPersQueue : public NKeyValue::TKeyValueFlat { void FlushRequests(bool force, const TActorContext& ctx); void FlushShardData(bool force, const TActorContext& ctx); - TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> MakeRegisterMessageGroup( - const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd, - NPersQueue::NErrorCode::EErrorCode& code, TString& error) const; - - TMaybe<TEvPQ::TEvDeregisterMessageGroup::TBody> MakeDeregisterMessageGroup( - const NKikimrClient::TPersQueuePartitionRequest::TCmdDeregisterMessageGroup& cmd, - NPersQueue::NErrorCode::EErrorCode& code, TString& error) const; - + TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> MakeRegisterMessageGroup( + const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd, + NPersQueue::NErrorCode::EErrorCode& code, TString& error) const; + + TMaybe<TEvPQ::TEvDeregisterMessageGroup::TBody> MakeDeregisterMessageGroup( + const NKikimrClient::TPersQueuePartitionRequest::TCmdDeregisterMessageGroup& cmd, + NPersQueue::NErrorCode::EErrorCode& code, TString& error) const; + //client request void Handle(TEvPersQueue::TEvRequest::TPtr& ev, const TActorContext& ctx); #define DESCRIBE_HANDLE(A) void A(const ui64 responseCookie, const TActorId& partActor, \ @@ -108,9 +108,9 @@ class TPersQueue : public NKeyValue::TKeyValueFlat { DESCRIBE_HANDLE(HandleWriteRequest) DESCRIBE_HANDLE(HandleUpdateWriteTimestampRequest) DESCRIBE_HANDLE(HandleReadRequest) - DESCRIBE_HANDLE(HandleRegisterMessageGroupRequest) - DESCRIBE_HANDLE(HandleDeregisterMessageGroupRequest) - DESCRIBE_HANDLE(HandleSplitMessageGroupRequest) + DESCRIBE_HANDLE(HandleRegisterMessageGroupRequest) + DESCRIBE_HANDLE(HandleDeregisterMessageGroupRequest) + DESCRIBE_HANDLE(HandleSplitMessageGroupRequest) #undef DESCRIBE_HANDLE #define DESCRIBE_HANDLE_WITH_SENDER(A) void A(const ui64 responseCookie, const TActorId& partActor, \ const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx,\ @@ -148,7 +148,7 @@ private: TString TopicPath; bool LocalDC; TString DCId; - TVector<NScheme::TTypeId> KeySchema; + TVector<NScheme::TTypeId> KeySchema; NKikimrPQ::TPQTabletConfig Config; NKikimrPQ::ETabletState TabletState; diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp index cdbffb0cadb..13a61229105 100644 --- a/ydb/core/persqueue/pq_ut.cpp +++ b/ydb/core/persqueue/pq_ut.cpp @@ -1280,7 +1280,7 @@ Y_UNIT_TEST(TestSourceIdDropBySourceIdCount) { } CmdWrite(0,"sourceid0", data, tc, false); Cout << "written sourceid0" << Endl; - PQGetPartInfo(2000, 3007, tc); + PQGetPartInfo(2000, 3007, tc); }); } diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h index 4c9de040542..3db93bc1047 100644 --- a/ydb/core/persqueue/pq_ut.h +++ b/ydb/core/persqueue/pq_ut.h @@ -223,7 +223,7 @@ void PQTabletPrepare(ui32 mcip, ui64 msip, ui32 deleteTime, const TVector<std::p THolder<TEvPersQueue::TEvUpdateConfig> request(new TEvPersQueue::TEvUpdateConfig()); for (i32 i = 0; i < partitions; ++i) { - request->Record.MutableTabletConfig()->AddPartitionIds(i); + request->Record.MutableTabletConfig()->AddPartitionIds(i); } request->Record.MutableTabletConfig()->SetCacheSize(10*1024*1024); request->Record.SetTxId(12345); diff --git a/ydb/core/persqueue/sourceid.cpp b/ydb/core/persqueue/sourceid.cpp index 39856ce5a15..a63b2f6f705 100644 --- a/ydb/core/persqueue/sourceid.cpp +++ b/ydb/core/persqueue/sourceid.cpp @@ -1,429 +1,429 @@ #include "sourceid.h" #include "ownerinfo.h" - + #include <ydb/core/persqueue/partition.h> - -#include <util/generic/size_literals.h> - -#include <algorithm> + +#include <util/generic/size_literals.h> + +#include <algorithm> namespace NKikimr { namespace NPQ { -static constexpr ui64 MAX_DELETE_COMMAND_SIZE = 10_MB; -static constexpr ui64 MAX_DELETE_COMMAND_COUNT = 1000; - -template <typename T> -T ReadAs(const TString& data, ui32& pos) { - auto result = *((T*)(data.c_str() + pos)); - pos += sizeof(T); - return result; -} - -template <typename T> -T ReadAs(const TString& data, ui32& pos, const T& default_) { - if (pos + sizeof(T) <= data.size()) { - return ReadAs<T>(data, pos); - } else { - return default_; - } -} - -template <typename T> -void Write(T value, TBuffer& data, ui32& pos) { - memcpy(data.Data() + pos, &value, sizeof(T)); - pos += sizeof(T); -} - -TSourceIdInfo::EState TSourceIdInfo::ConvertState(NKikimrPQ::TMessageGroupInfo::EState value) { - switch (value) { - case NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED: - return TSourceIdInfo::EState::Registered; - case NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION: - return TSourceIdInfo::EState::PendingRegistration; - default: - return TSourceIdInfo::EState::Unknown; - } -} - -NKikimrPQ::TMessageGroupInfo::EState TSourceIdInfo::ConvertState(TSourceIdInfo::EState value) { - switch (value) { - case TSourceIdInfo::EState::Registered: - return NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED; - case TSourceIdInfo::EState::PendingRegistration: - return NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION; - case TSourceIdInfo::EState::Unknown: - return NKikimrPQ::TMessageGroupInfo::STATE_UNKNOWN; - } -} - -void FillWrite(const TKeyPrefix& key, const TBuffer& data, NKikimrClient::TKeyValueRequest::TCmdWrite& cmd) { - cmd.SetKey(key.Data(), key.Size()); - cmd.SetValue(data.Data(), data.Size()); - cmd.SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); -} - -void FillDelete(const TKeyPrefix& key, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { - auto& range = *cmd.MutableRange(); - range.SetFrom(key.Data(), key.Size()); - range.SetIncludeFrom(true); - range.SetTo(key.Data(), key.Size()); - range.SetIncludeTo(true); -} - -void FillDelete(ui32 partition, const TString& sourceId, TKeyPrefix::EMark mark, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { - TKeyPrefix key(TKeyPrefix::TypeInfo, partition, mark); - key.Append(sourceId.c_str(), sourceId.size()); - FillDelete(key, cmd); -} - -void FillDelete(ui32 partition, const TString& sourceId, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { - FillDelete(partition, sourceId, TKeyPrefix::MarkProtoSourceId, cmd); -} - -TSourceIdInfo::TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs) - : SeqNo(seqNo) - , Offset(offset) - , WriteTimestamp(createTs) - , CreateTimestamp(createTs) -{ -} - -TSourceIdInfo::TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe<TPartitionKeyRange>&& keyRange, bool isInSplit) - : SeqNo(seqNo) - , Offset(offset) - , CreateTimestamp(createTs) - , Explicit(true) - , KeyRange(std::move(keyRange)) -{ - if (isInSplit) { - State = EState::PendingRegistration; - } -} - -TSourceIdInfo TSourceIdInfo::Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const { - auto copy = *this; - copy.SeqNo = seqNo; - copy.Offset = offset; - copy.WriteTimestamp = writeTs; - - return copy; -} - -TSourceIdInfo TSourceIdInfo::Parse(const TString& data, TInstant now) { - Y_VERIFY(data.size() >= 2 * sizeof(ui64), "Data must contain at least SeqNo & Offset"); - ui32 pos = 0; - - TSourceIdInfo result; - result.SeqNo = ReadAs<ui64>(data, pos); - result.Offset = ReadAs<ui64>(data, pos); - result.WriteTimestamp = TInstant::MilliSeconds(ReadAs<ui64>(data, pos, now.MilliSeconds())); - result.CreateTimestamp = TInstant::MilliSeconds(ReadAs<ui64>(data, pos, now.MilliSeconds())); - - Y_VERIFY(result.SeqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, result.SeqNo); - Y_VERIFY(result.Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, result.Offset); - - return result; -} - -void TSourceIdInfo::Serialize(TBuffer& data) const { - Y_VERIFY(!Explicit); - Y_VERIFY(!KeyRange); - - data.Resize(4 * sizeof(ui64)); - ui32 pos = 0; - - Write<ui64>(SeqNo, data, pos); - Write<ui64>(Offset, data, pos); - Write<ui64>(WriteTimestamp.MilliSeconds(), data, pos); - Write<ui64>(CreateTimestamp.MilliSeconds(), data, pos); -} - -TSourceIdInfo TSourceIdInfo::Parse(const NKikimrPQ::TMessageGroupInfo& proto) { - TSourceIdInfo result; - result.SeqNo = proto.GetSeqNo(); - result.Offset = proto.GetOffset(); - result.WriteTimestamp = TInstant::FromValue(proto.GetWriteTimestamp()); - result.CreateTimestamp = TInstant::FromValue(proto.GetCreateTimestamp()); - result.Explicit = proto.GetExplicit(); - if (result.Explicit) { - result.State = ConvertState(proto.GetState()); - } - if (proto.HasKeyRange()) { - result.KeyRange = TPartitionKeyRange::Parse(proto.GetKeyRange()); - } - - return result; -} - -void TSourceIdInfo::Serialize(NKikimrPQ::TMessageGroupInfo& proto) const { - proto.SetSeqNo(SeqNo); - proto.SetOffset(Offset); - proto.SetWriteTimestamp(WriteTimestamp.GetValue()); - proto.SetCreateTimestamp(CreateTimestamp.GetValue()); - proto.SetExplicit(Explicit); - if (Explicit) { - proto.SetState(ConvertState(State)); - } - if (KeyRange) { - KeyRange->Serialize(*proto.MutableKeyRange()); - } -} - -bool TSourceIdInfo::operator==(const TSourceIdInfo& rhs) const { - return SeqNo == rhs.SeqNo - && Offset == rhs.Offset - && WriteTimestamp == rhs.WriteTimestamp - && CreateTimestamp == rhs.CreateTimestamp - && Explicit == rhs.Explicit - && State == rhs.State; -} - -void TSourceIdInfo::Out(IOutputStream& out) const { - out << "{" - << " SeqNo: " << SeqNo - << " Offset: " << Offset - << " WriteTimestamp: " << WriteTimestamp.GetValue() - << " CreateTimestamp: " << CreateTimestamp.GetValue() - << " Explicit: " << (Explicit ? "true" : "false") - << " State: " << State - << " }"; -} - -bool TSourceIdInfo::IsExpired(TDuration ttl, TInstant now) const { - Y_VERIFY_DEBUG(!Explicit); - return !Explicit && ((WriteTimestamp + ttl) <= now); -} - -/// TSourceIdStorage -void TSourceIdStorage::DeregisterSourceId(const TString& sourceId) { - auto it = InMemorySourceIds.find(sourceId); - if (it == InMemorySourceIds.end()) { - return; - } - - SourceIdsByOffset.erase(std::make_pair(it->second.Offset, sourceId)); - InMemorySourceIds.erase(it); - - auto jt = SourceIdOwners.find(sourceId); - if (jt != SourceIdOwners.end()) { - OwnersToDrop.push_back(jt->second); - SourceIdOwners.erase(jt); - } -} - -bool TSourceIdStorage::DropOldSourceIds(TEvKeyValue::TEvRequest* request, TInstant now, ui64 startOffset, ui32 partition, const NKikimrPQ::TPartitionConfig& config) { - TVector<std::pair<ui64, TString>> toDelOffsets; - - ui64 maxDeleteSourceIds = 0; - if (InMemorySourceIds.size() > config.GetSourceIdMaxCounts()) { - maxDeleteSourceIds = InMemorySourceIds.size() - config.GetSourceIdMaxCounts(); - maxDeleteSourceIds = Min(maxDeleteSourceIds, MAX_DELETE_COMMAND_COUNT); - } - - Y_VERIFY(maxDeleteSourceIds <= InMemorySourceIds.size()); - - const auto ttl = TDuration::Seconds(config.GetSourceIdLifetimeSeconds()); - ui32 size = request->Record.ByteSize(); - - for (const auto& [offset, sourceId] : SourceIdsByOffset) { - if (offset >= startOffset && toDelOffsets.size() >= maxDeleteSourceIds) { - break; - } - - auto it = InMemorySourceIds.find(sourceId); - Y_VERIFY(it != InMemorySourceIds.end()); - if (it->second.Explicit) { - continue; - } - - if (toDelOffsets.size() < maxDeleteSourceIds || it->second.IsExpired(ttl, now)) { - toDelOffsets.emplace_back(offset, sourceId); - - bool reachedLimit = false; - for (const auto mark : {TKeyPrefix::MarkSourceId, TKeyPrefix::MarkProtoSourceId}) { - auto& cmd = *request->Record.AddCmdDeleteRange(); - FillDelete(partition, sourceId, mark, cmd); - - size += cmd.ByteSize(); - if (size >= MAX_DELETE_COMMAND_SIZE || toDelOffsets.size() >= MAX_DELETE_COMMAND_COUNT) { - LOG_INFO_S(*TlsActivationContext, NKikimrServices::PERSQUEUE, "DropOldSourceIds reached proto size limit" - << ": size# " << size - << ", count# " << toDelOffsets.size()); - reachedLimit = true; - break; - } +static constexpr ui64 MAX_DELETE_COMMAND_SIZE = 10_MB; +static constexpr ui64 MAX_DELETE_COMMAND_COUNT = 1000; + +template <typename T> +T ReadAs(const TString& data, ui32& pos) { + auto result = *((T*)(data.c_str() + pos)); + pos += sizeof(T); + return result; +} + +template <typename T> +T ReadAs(const TString& data, ui32& pos, const T& default_) { + if (pos + sizeof(T) <= data.size()) { + return ReadAs<T>(data, pos); + } else { + return default_; + } +} + +template <typename T> +void Write(T value, TBuffer& data, ui32& pos) { + memcpy(data.Data() + pos, &value, sizeof(T)); + pos += sizeof(T); +} + +TSourceIdInfo::EState TSourceIdInfo::ConvertState(NKikimrPQ::TMessageGroupInfo::EState value) { + switch (value) { + case NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED: + return TSourceIdInfo::EState::Registered; + case NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION: + return TSourceIdInfo::EState::PendingRegistration; + default: + return TSourceIdInfo::EState::Unknown; + } +} + +NKikimrPQ::TMessageGroupInfo::EState TSourceIdInfo::ConvertState(TSourceIdInfo::EState value) { + switch (value) { + case TSourceIdInfo::EState::Registered: + return NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED; + case TSourceIdInfo::EState::PendingRegistration: + return NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION; + case TSourceIdInfo::EState::Unknown: + return NKikimrPQ::TMessageGroupInfo::STATE_UNKNOWN; + } +} + +void FillWrite(const TKeyPrefix& key, const TBuffer& data, NKikimrClient::TKeyValueRequest::TCmdWrite& cmd) { + cmd.SetKey(key.Data(), key.Size()); + cmd.SetValue(data.Data(), data.Size()); + cmd.SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); +} + +void FillDelete(const TKeyPrefix& key, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { + auto& range = *cmd.MutableRange(); + range.SetFrom(key.Data(), key.Size()); + range.SetIncludeFrom(true); + range.SetTo(key.Data(), key.Size()); + range.SetIncludeTo(true); +} + +void FillDelete(ui32 partition, const TString& sourceId, TKeyPrefix::EMark mark, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { + TKeyPrefix key(TKeyPrefix::TypeInfo, partition, mark); + key.Append(sourceId.c_str(), sourceId.size()); + FillDelete(key, cmd); +} + +void FillDelete(ui32 partition, const TString& sourceId, NKikimrClient::TKeyValueRequest::TCmdDeleteRange& cmd) { + FillDelete(partition, sourceId, TKeyPrefix::MarkProtoSourceId, cmd); +} + +TSourceIdInfo::TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs) + : SeqNo(seqNo) + , Offset(offset) + , WriteTimestamp(createTs) + , CreateTimestamp(createTs) +{ +} + +TSourceIdInfo::TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe<TPartitionKeyRange>&& keyRange, bool isInSplit) + : SeqNo(seqNo) + , Offset(offset) + , CreateTimestamp(createTs) + , Explicit(true) + , KeyRange(std::move(keyRange)) +{ + if (isInSplit) { + State = EState::PendingRegistration; + } +} + +TSourceIdInfo TSourceIdInfo::Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const { + auto copy = *this; + copy.SeqNo = seqNo; + copy.Offset = offset; + copy.WriteTimestamp = writeTs; + + return copy; +} + +TSourceIdInfo TSourceIdInfo::Parse(const TString& data, TInstant now) { + Y_VERIFY(data.size() >= 2 * sizeof(ui64), "Data must contain at least SeqNo & Offset"); + ui32 pos = 0; + + TSourceIdInfo result; + result.SeqNo = ReadAs<ui64>(data, pos); + result.Offset = ReadAs<ui64>(data, pos); + result.WriteTimestamp = TInstant::MilliSeconds(ReadAs<ui64>(data, pos, now.MilliSeconds())); + result.CreateTimestamp = TInstant::MilliSeconds(ReadAs<ui64>(data, pos, now.MilliSeconds())); + + Y_VERIFY(result.SeqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, result.SeqNo); + Y_VERIFY(result.Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, result.Offset); + + return result; +} + +void TSourceIdInfo::Serialize(TBuffer& data) const { + Y_VERIFY(!Explicit); + Y_VERIFY(!KeyRange); + + data.Resize(4 * sizeof(ui64)); + ui32 pos = 0; + + Write<ui64>(SeqNo, data, pos); + Write<ui64>(Offset, data, pos); + Write<ui64>(WriteTimestamp.MilliSeconds(), data, pos); + Write<ui64>(CreateTimestamp.MilliSeconds(), data, pos); +} + +TSourceIdInfo TSourceIdInfo::Parse(const NKikimrPQ::TMessageGroupInfo& proto) { + TSourceIdInfo result; + result.SeqNo = proto.GetSeqNo(); + result.Offset = proto.GetOffset(); + result.WriteTimestamp = TInstant::FromValue(proto.GetWriteTimestamp()); + result.CreateTimestamp = TInstant::FromValue(proto.GetCreateTimestamp()); + result.Explicit = proto.GetExplicit(); + if (result.Explicit) { + result.State = ConvertState(proto.GetState()); + } + if (proto.HasKeyRange()) { + result.KeyRange = TPartitionKeyRange::Parse(proto.GetKeyRange()); + } + + return result; +} + +void TSourceIdInfo::Serialize(NKikimrPQ::TMessageGroupInfo& proto) const { + proto.SetSeqNo(SeqNo); + proto.SetOffset(Offset); + proto.SetWriteTimestamp(WriteTimestamp.GetValue()); + proto.SetCreateTimestamp(CreateTimestamp.GetValue()); + proto.SetExplicit(Explicit); + if (Explicit) { + proto.SetState(ConvertState(State)); + } + if (KeyRange) { + KeyRange->Serialize(*proto.MutableKeyRange()); + } +} + +bool TSourceIdInfo::operator==(const TSourceIdInfo& rhs) const { + return SeqNo == rhs.SeqNo + && Offset == rhs.Offset + && WriteTimestamp == rhs.WriteTimestamp + && CreateTimestamp == rhs.CreateTimestamp + && Explicit == rhs.Explicit + && State == rhs.State; +} + +void TSourceIdInfo::Out(IOutputStream& out) const { + out << "{" + << " SeqNo: " << SeqNo + << " Offset: " << Offset + << " WriteTimestamp: " << WriteTimestamp.GetValue() + << " CreateTimestamp: " << CreateTimestamp.GetValue() + << " Explicit: " << (Explicit ? "true" : "false") + << " State: " << State + << " }"; +} + +bool TSourceIdInfo::IsExpired(TDuration ttl, TInstant now) const { + Y_VERIFY_DEBUG(!Explicit); + return !Explicit && ((WriteTimestamp + ttl) <= now); +} + +/// TSourceIdStorage +void TSourceIdStorage::DeregisterSourceId(const TString& sourceId) { + auto it = InMemorySourceIds.find(sourceId); + if (it == InMemorySourceIds.end()) { + return; + } + + SourceIdsByOffset.erase(std::make_pair(it->second.Offset, sourceId)); + InMemorySourceIds.erase(it); + + auto jt = SourceIdOwners.find(sourceId); + if (jt != SourceIdOwners.end()) { + OwnersToDrop.push_back(jt->second); + SourceIdOwners.erase(jt); + } +} + +bool TSourceIdStorage::DropOldSourceIds(TEvKeyValue::TEvRequest* request, TInstant now, ui64 startOffset, ui32 partition, const NKikimrPQ::TPartitionConfig& config) { + TVector<std::pair<ui64, TString>> toDelOffsets; + + ui64 maxDeleteSourceIds = 0; + if (InMemorySourceIds.size() > config.GetSourceIdMaxCounts()) { + maxDeleteSourceIds = InMemorySourceIds.size() - config.GetSourceIdMaxCounts(); + maxDeleteSourceIds = Min(maxDeleteSourceIds, MAX_DELETE_COMMAND_COUNT); + } + + Y_VERIFY(maxDeleteSourceIds <= InMemorySourceIds.size()); + + const auto ttl = TDuration::Seconds(config.GetSourceIdLifetimeSeconds()); + ui32 size = request->Record.ByteSize(); + + for (const auto& [offset, sourceId] : SourceIdsByOffset) { + if (offset >= startOffset && toDelOffsets.size() >= maxDeleteSourceIds) { + break; + } + + auto it = InMemorySourceIds.find(sourceId); + Y_VERIFY(it != InMemorySourceIds.end()); + if (it->second.Explicit) { + continue; + } + + if (toDelOffsets.size() < maxDeleteSourceIds || it->second.IsExpired(ttl, now)) { + toDelOffsets.emplace_back(offset, sourceId); + + bool reachedLimit = false; + for (const auto mark : {TKeyPrefix::MarkSourceId, TKeyPrefix::MarkProtoSourceId}) { + auto& cmd = *request->Record.AddCmdDeleteRange(); + FillDelete(partition, sourceId, mark, cmd); + + size += cmd.ByteSize(); + if (size >= MAX_DELETE_COMMAND_SIZE || toDelOffsets.size() >= MAX_DELETE_COMMAND_COUNT) { + LOG_INFO_S(*TlsActivationContext, NKikimrServices::PERSQUEUE, "DropOldSourceIds reached proto size limit" + << ": size# " << size + << ", count# " << toDelOffsets.size()); + reachedLimit = true; + break; + } } - if (reachedLimit) { - break; // Check here size to prevent crashing in protobuf verify -max size is 25Mb's and no more then 100K operations - // but even 10Mbs sounds too big here. - // Rest of SourceIds will be deleted in next DropOldSourceIds calls, whitch will be made - // right after complete of current deletion. + if (reachedLimit) { + break; // Check here size to prevent crashing in protobuf verify -max size is 25Mb's and no more then 100K operations + // but even 10Mbs sounds too big here. + // Rest of SourceIds will be deleted in next DropOldSourceIds calls, whitch will be made + // right after complete of current deletion. } - } else { - // there are no space for new sourcID to delete, stop check sourceIds - break; + } else { + // there are no space for new sourcID to delete, stop check sourceIds + break; + } + } + + for (auto& t : toDelOffsets) { + // delete sourceId from memory + size_t res = InMemorySourceIds.erase(t.second); + Y_VERIFY(res == 1); + // delete sourceID from offsets + res = SourceIdsByOffset.erase(t); + Y_VERIFY(res == 1); + // save owners to drop and delete records from map + auto it = SourceIdOwners.find(t.second); + if (it != SourceIdOwners.end()) { + OwnersToDrop.push_back(it->second); + SourceIdOwners.erase(it); } } - for (auto& t : toDelOffsets) { - // delete sourceId from memory - size_t res = InMemorySourceIds.erase(t.second); - Y_VERIFY(res == 1); - // delete sourceID from offsets - res = SourceIdsByOffset.erase(t); - Y_VERIFY(res == 1); - // save owners to drop and delete records from map - auto it = SourceIdOwners.find(t.second); - if (it != SourceIdOwners.end()) { - OwnersToDrop.push_back(it->second); - SourceIdOwners.erase(it); - } - } - - return !toDelOffsets.empty(); -} - -void TSourceIdStorage::LoadSourceIdInfo(const TString& key, const TString& data, TInstant now) { - Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); - - const auto mark = TKeyPrefix::EMark(key[TKeyPrefix::MarkPosition()]); - switch (mark) { - case TKeyPrefix::MarkSourceId: - return LoadRawSourceIdInfo(key, data, now); - case TKeyPrefix::MarkProtoSourceId: - return LoadProtoSourceIdInfo(key, data); - default: - Y_FAIL_S("Unexpected mark: " << (char)mark); - } -} - -void TSourceIdStorage::LoadRawSourceIdInfo(const TString& key, const TString& data, TInstant now) { - Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); - Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkSourceId); - - RegisterSourceIdInfo(key.substr(TKeyPrefix::MarkedSize()), TSourceIdInfo::Parse(data, now), true); -} - -void TSourceIdStorage::LoadProtoSourceIdInfo(const TString& key, const TString& data) { - Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); - Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId); - - NKikimrPQ::TMessageGroupInfo proto; - const bool ok = proto.ParseFromString(data); - Y_VERIFY(ok); - - RegisterSourceIdInfo(key.substr(TKeyPrefix::MarkedSize()), TSourceIdInfo::Parse(proto), true); -} - -void TSourceIdStorage::RegisterSourceIdInfo(const TString& sourceId, TSourceIdInfo&& sourceIdInfo, bool load) { - auto it = InMemorySourceIds.find(sourceId); - if (it != InMemorySourceIds.end()) { - Y_VERIFY(!load); - const auto res = SourceIdsByOffset.erase(std::make_pair(it->second.Offset, sourceId)); - Y_VERIFY(res == 1); - } - - const auto offset = sourceIdInfo.Offset; - InMemorySourceIds[sourceId] = std::move(sourceIdInfo); - - const bool res = SourceIdsByOffset.emplace(offset, sourceId).second; - Y_VERIFY(res); -} - -void TSourceIdStorage::RegisterSourceIdOwner(const TString& sourceId, const TStringBuf& ownerCookie) { - if (ownerCookie == "default") { - // cookie for legacy http protocol - skip it, we use one cookie for all write sessions - return; + return !toDelOffsets.empty(); +} + +void TSourceIdStorage::LoadSourceIdInfo(const TString& key, const TString& data, TInstant now) { + Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); + + const auto mark = TKeyPrefix::EMark(key[TKeyPrefix::MarkPosition()]); + switch (mark) { + case TKeyPrefix::MarkSourceId: + return LoadRawSourceIdInfo(key, data, now); + case TKeyPrefix::MarkProtoSourceId: + return LoadProtoSourceIdInfo(key, data); + default: + Y_FAIL_S("Unexpected mark: " << (char)mark); + } +} + +void TSourceIdStorage::LoadRawSourceIdInfo(const TString& key, const TString& data, TInstant now) { + Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); + Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkSourceId); + + RegisterSourceIdInfo(key.substr(TKeyPrefix::MarkedSize()), TSourceIdInfo::Parse(data, now), true); +} + +void TSourceIdStorage::LoadProtoSourceIdInfo(const TString& key, const TString& data) { + Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); + Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId); + + NKikimrPQ::TMessageGroupInfo proto; + const bool ok = proto.ParseFromString(data); + Y_VERIFY(ok); + + RegisterSourceIdInfo(key.substr(TKeyPrefix::MarkedSize()), TSourceIdInfo::Parse(proto), true); +} + +void TSourceIdStorage::RegisterSourceIdInfo(const TString& sourceId, TSourceIdInfo&& sourceIdInfo, bool load) { + auto it = InMemorySourceIds.find(sourceId); + if (it != InMemorySourceIds.end()) { + Y_VERIFY(!load); + const auto res = SourceIdsByOffset.erase(std::make_pair(it->second.Offset, sourceId)); + Y_VERIFY(res == 1); } - // owner cookie could be deleted in main object - so we should copy it - SourceIdOwners[sourceId] = ownerCookie; -} - -void TSourceIdStorage::MarkOwnersForDeletedSourceId(THashMap<TString, TOwnerInfo>& owners) { - for (auto& sourceid : OwnersToDrop) { - auto it = owners.find(sourceid); - if (it != owners.end()) { - it->second.SourceIdDeleted = true; + + const auto offset = sourceIdInfo.Offset; + InMemorySourceIds[sourceId] = std::move(sourceIdInfo); + + const bool res = SourceIdsByOffset.emplace(offset, sourceId).second; + Y_VERIFY(res); +} + +void TSourceIdStorage::RegisterSourceIdOwner(const TString& sourceId, const TStringBuf& ownerCookie) { + if (ownerCookie == "default") { + // cookie for legacy http protocol - skip it, we use one cookie for all write sessions + return; + } + // owner cookie could be deleted in main object - so we should copy it + SourceIdOwners[sourceId] = ownerCookie; +} + +void TSourceIdStorage::MarkOwnersForDeletedSourceId(THashMap<TString, TOwnerInfo>& owners) { + for (auto& sourceid : OwnersToDrop) { + auto it = owners.find(sourceid); + if (it != owners.end()) { + it->second.SourceIdDeleted = true; } } - OwnersToDrop.clear(); -} + OwnersToDrop.clear(); +} -TInstant TSourceIdStorage::MinAvailableTimestamp(TInstant now) const { - TInstant ds = now; - if (!SourceIdsByOffset.empty()) { - auto it = InMemorySourceIds.find(SourceIdsByOffset.begin()->second); - Y_VERIFY(it != InMemorySourceIds.end()); - ds = Min(ds, it->second.WriteTimestamp); +TInstant TSourceIdStorage::MinAvailableTimestamp(TInstant now) const { + TInstant ds = now; + if (!SourceIdsByOffset.empty()) { + auto it = InMemorySourceIds.find(SourceIdsByOffset.begin()->second); + Y_VERIFY(it != InMemorySourceIds.end()); + ds = Min(ds, it->second.WriteTimestamp); } - - return ds; -} - -/// TSourceIdWriter -TSourceIdWriter::TSourceIdWriter(ESourceIdFormat format) - : Format(format) -{ -} - -void TSourceIdWriter::DeregisterSourceId(const TString& sourceId) { - Deregistrations.insert(sourceId); -} - -void TSourceIdWriter::Clear() { - Registrations.clear(); - Deregistrations.clear(); -} - -void TSourceIdWriter::FillKeyAndData(ESourceIdFormat format, const TString& sourceId, const TSourceIdInfo& sourceIdInfo, TKeyPrefix& key, TBuffer& data) { - key.Resize(TKeyPrefix::MarkedSize()); - key.Append(sourceId.c_str(), sourceId.size()); - - switch (format) { - case ESourceIdFormat::Raw: - return FillRawData(sourceIdInfo, data); - case ESourceIdFormat::Proto: - return FillProtoData(sourceIdInfo, data); - } -} - -void TSourceIdWriter::FillRawData(const TSourceIdInfo& sourceIdInfo, TBuffer& data) { - sourceIdInfo.Serialize(data); -} - -void TSourceIdWriter::FillProtoData(const TSourceIdInfo& sourceIdInfo, TBuffer& data) { - NKikimrPQ::TMessageGroupInfo proto; - sourceIdInfo.Serialize(proto); - - const TString serialized = proto.SerializeAsString(); - data.Assign(serialized.c_str(), serialized.size()); -} - -TKeyPrefix::EMark TSourceIdWriter::FormatToMark(ESourceIdFormat format) { - switch (format) { - case ESourceIdFormat::Raw: - return TKeyPrefix::MarkSourceId; - case ESourceIdFormat::Proto: - return TKeyPrefix::MarkProtoSourceId; - } -} - -void TSourceIdWriter::FillRequest(TEvKeyValue::TEvRequest* request, ui32 partition) { - TKeyPrefix key(TKeyPrefix::TypeInfo, partition, FormatToMark(Format)); - TBuffer data; - - for (const auto& [sourceId, sourceIdInfo] : Registrations) { - FillKeyAndData(Format, sourceId, sourceIdInfo, key, data); - FillWrite(key, data, *request->Record.AddCmdWrite()); + + return ds; +} + +/// TSourceIdWriter +TSourceIdWriter::TSourceIdWriter(ESourceIdFormat format) + : Format(format) +{ +} + +void TSourceIdWriter::DeregisterSourceId(const TString& sourceId) { + Deregistrations.insert(sourceId); +} + +void TSourceIdWriter::Clear() { + Registrations.clear(); + Deregistrations.clear(); +} + +void TSourceIdWriter::FillKeyAndData(ESourceIdFormat format, const TString& sourceId, const TSourceIdInfo& sourceIdInfo, TKeyPrefix& key, TBuffer& data) { + key.Resize(TKeyPrefix::MarkedSize()); + key.Append(sourceId.c_str(), sourceId.size()); + + switch (format) { + case ESourceIdFormat::Raw: + return FillRawData(sourceIdInfo, data); + case ESourceIdFormat::Proto: + return FillProtoData(sourceIdInfo, data); + } +} + +void TSourceIdWriter::FillRawData(const TSourceIdInfo& sourceIdInfo, TBuffer& data) { + sourceIdInfo.Serialize(data); +} + +void TSourceIdWriter::FillProtoData(const TSourceIdInfo& sourceIdInfo, TBuffer& data) { + NKikimrPQ::TMessageGroupInfo proto; + sourceIdInfo.Serialize(proto); + + const TString serialized = proto.SerializeAsString(); + data.Assign(serialized.c_str(), serialized.size()); +} + +TKeyPrefix::EMark TSourceIdWriter::FormatToMark(ESourceIdFormat format) { + switch (format) { + case ESourceIdFormat::Raw: + return TKeyPrefix::MarkSourceId; + case ESourceIdFormat::Proto: + return TKeyPrefix::MarkProtoSourceId; + } +} + +void TSourceIdWriter::FillRequest(TEvKeyValue::TEvRequest* request, ui32 partition) { + TKeyPrefix key(TKeyPrefix::TypeInfo, partition, FormatToMark(Format)); + TBuffer data; + + for (const auto& [sourceId, sourceIdInfo] : Registrations) { + FillKeyAndData(Format, sourceId, sourceIdInfo, key, data); + FillWrite(key, data, *request->Record.AddCmdWrite()); + } + + for (const auto& sourceId : Deregistrations) { + FillDelete(partition, sourceId, *request->Record.AddCmdDeleteRange()); } - - for (const auto& sourceId : Deregistrations) { - FillDelete(partition, sourceId, *request->Record.AddCmdDeleteRange()); - } -} +} } // NPQ } // NKikimr diff --git a/ydb/core/persqueue/sourceid.h b/ydb/core/persqueue/sourceid.h index 7c48ae45474..62b9c2834e2 100644 --- a/ydb/core/persqueue/sourceid.h +++ b/ydb/core/persqueue/sourceid.h @@ -1,130 +1,130 @@ #pragma once - + #include <ydb/core/keyvalue/keyvalue_events.h> #include <ydb/core/persqueue/key.h> #include <ydb/core/persqueue/ownerinfo.h> #include <ydb/core/persqueue/partition_key_range/partition_key_range.h> #include <ydb/core/protos/pqconfig.pb.h> - + #include <util/generic/set.h> namespace NKikimr { namespace NPQ { -enum class ESourceIdFormat: ui8 { - Raw = 0, - Proto = 1, -}; - -struct TSourceIdInfo { - enum class EState { - Unknown, - Registered, - PendingRegistration, - }; - - ui64 SeqNo = 0; - ui64 Offset = 0; - TInstant WriteTimestamp; - TInstant CreateTimestamp; - bool Explicit = false; - TMaybe<TPartitionKeyRange> KeyRange; - EState State = EState::Registered; - - TSourceIdInfo() = default; - TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs); - TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe<TPartitionKeyRange>&& keyRange, bool isInSplit = false); - - TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const; - - static EState ConvertState(NKikimrPQ::TMessageGroupInfo::EState value); - static NKikimrPQ::TMessageGroupInfo::EState ConvertState(EState value); - - // Raw format - static TSourceIdInfo Parse(const TString& data, TInstant now); - void Serialize(TBuffer& data) const; - - // Proto format - static TSourceIdInfo Parse(const NKikimrPQ::TMessageGroupInfo& proto); - void Serialize(NKikimrPQ::TMessageGroupInfo& proto) const; - - bool operator==(const TSourceIdInfo& rhs) const; - void Out(IOutputStream& out) const; - - bool IsExpired(TDuration ttl, TInstant now) const; - -}; // TSourceIdInfo - -using TSourceIdMap = THashMap<TString, TSourceIdInfo>; - -class TSourceIdStorage { -public: - const TSourceIdMap& GetInMemorySourceIds() const { - return InMemorySourceIds; - } - - template <typename... Args> - void RegisterSourceId(const TString& sourceId, Args&&... args) { - RegisterSourceIdInfo(sourceId, TSourceIdInfo(std::forward<Args>(args)...), false); - } - - void DeregisterSourceId(const TString& sourceId); - - void LoadSourceIdInfo(const TString& key, const TString& data, TInstant now); - bool DropOldSourceIds(TEvKeyValue::TEvRequest* request, TInstant now, ui64 startOffset, ui32 partition, const NKikimrPQ::TPartitionConfig& config); - - void RegisterSourceIdOwner(const TString& sourceId, const TStringBuf& ownerCookie); - void MarkOwnersForDeletedSourceId(THashMap<TString, TOwnerInfo>& owners); - - TInstant MinAvailableTimestamp(TInstant now) const; - -private: - void LoadRawSourceIdInfo(const TString& key, const TString& data, TInstant now); - void LoadProtoSourceIdInfo(const TString& key, const TString& data); - void RegisterSourceIdInfo(const TString& sourceId, TSourceIdInfo&& sourceIdInfo, bool load); - -private: - TSourceIdMap InMemorySourceIds; - THashMap<TString, TString> SourceIdOwners; - TVector<TString> OwnersToDrop; - TSet<std::pair<ui64, TString>> SourceIdsByOffset; - -}; // TSourceIdStorage - -class TSourceIdWriter { -public: - explicit TSourceIdWriter(ESourceIdFormat format); - - const TSourceIdMap& GetSourceIdsToWrite() const { - return Registrations; - } - - template <typename... Args> - void RegisterSourceId(const TString& sourceId, Args&&... args) { - Registrations[sourceId] = TSourceIdInfo(std::forward<Args>(args)...); - } - - void DeregisterSourceId(const TString& sourceId); - void Clear(); - - void FillRequest(TEvKeyValue::TEvRequest* request, ui32 partition); - static void FillKeyAndData(ESourceIdFormat format, const TString& sourceId, const TSourceIdInfo& sourceIdInfo, TKeyPrefix& key, TBuffer& data); - -private: - static void FillRawData(const TSourceIdInfo& sourceIdInfo, TBuffer& data); - static void FillProtoData(const TSourceIdInfo& sourceIdInfo, TBuffer& data); - static TKeyPrefix::EMark FormatToMark(ESourceIdFormat format); - -private: - const ESourceIdFormat Format; - TSourceIdMap Registrations; - THashSet<TString> Deregistrations; - -}; // TSourceIdWriter - +enum class ESourceIdFormat: ui8 { + Raw = 0, + Proto = 1, +}; + +struct TSourceIdInfo { + enum class EState { + Unknown, + Registered, + PendingRegistration, + }; + + ui64 SeqNo = 0; + ui64 Offset = 0; + TInstant WriteTimestamp; + TInstant CreateTimestamp; + bool Explicit = false; + TMaybe<TPartitionKeyRange> KeyRange; + EState State = EState::Registered; + + TSourceIdInfo() = default; + TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs); + TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe<TPartitionKeyRange>&& keyRange, bool isInSplit = false); + + TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const; + + static EState ConvertState(NKikimrPQ::TMessageGroupInfo::EState value); + static NKikimrPQ::TMessageGroupInfo::EState ConvertState(EState value); + + // Raw format + static TSourceIdInfo Parse(const TString& data, TInstant now); + void Serialize(TBuffer& data) const; + + // Proto format + static TSourceIdInfo Parse(const NKikimrPQ::TMessageGroupInfo& proto); + void Serialize(NKikimrPQ::TMessageGroupInfo& proto) const; + + bool operator==(const TSourceIdInfo& rhs) const; + void Out(IOutputStream& out) const; + + bool IsExpired(TDuration ttl, TInstant now) const; + +}; // TSourceIdInfo + +using TSourceIdMap = THashMap<TString, TSourceIdInfo>; + +class TSourceIdStorage { +public: + const TSourceIdMap& GetInMemorySourceIds() const { + return InMemorySourceIds; + } + + template <typename... Args> + void RegisterSourceId(const TString& sourceId, Args&&... args) { + RegisterSourceIdInfo(sourceId, TSourceIdInfo(std::forward<Args>(args)...), false); + } + + void DeregisterSourceId(const TString& sourceId); + + void LoadSourceIdInfo(const TString& key, const TString& data, TInstant now); + bool DropOldSourceIds(TEvKeyValue::TEvRequest* request, TInstant now, ui64 startOffset, ui32 partition, const NKikimrPQ::TPartitionConfig& config); + + void RegisterSourceIdOwner(const TString& sourceId, const TStringBuf& ownerCookie); + void MarkOwnersForDeletedSourceId(THashMap<TString, TOwnerInfo>& owners); + + TInstant MinAvailableTimestamp(TInstant now) const; + +private: + void LoadRawSourceIdInfo(const TString& key, const TString& data, TInstant now); + void LoadProtoSourceIdInfo(const TString& key, const TString& data); + void RegisterSourceIdInfo(const TString& sourceId, TSourceIdInfo&& sourceIdInfo, bool load); + +private: + TSourceIdMap InMemorySourceIds; + THashMap<TString, TString> SourceIdOwners; + TVector<TString> OwnersToDrop; + TSet<std::pair<ui64, TString>> SourceIdsByOffset; + +}; // TSourceIdStorage + +class TSourceIdWriter { +public: + explicit TSourceIdWriter(ESourceIdFormat format); + + const TSourceIdMap& GetSourceIdsToWrite() const { + return Registrations; + } + + template <typename... Args> + void RegisterSourceId(const TString& sourceId, Args&&... args) { + Registrations[sourceId] = TSourceIdInfo(std::forward<Args>(args)...); + } + + void DeregisterSourceId(const TString& sourceId); + void Clear(); + + void FillRequest(TEvKeyValue::TEvRequest* request, ui32 partition); + static void FillKeyAndData(ESourceIdFormat format, const TString& sourceId, const TSourceIdInfo& sourceIdInfo, TKeyPrefix& key, TBuffer& data); + +private: + static void FillRawData(const TSourceIdInfo& sourceIdInfo, TBuffer& data); + static void FillProtoData(const TSourceIdInfo& sourceIdInfo, TBuffer& data); + static TKeyPrefix::EMark FormatToMark(ESourceIdFormat format); + +private: + const ESourceIdFormat Format; + TSourceIdMap Registrations; + THashSet<TString> Deregistrations; + +}; // TSourceIdWriter + } // NPQ } // NKikimr - -Y_DECLARE_OUT_SPEC(inline, NKikimr::NPQ::TSourceIdInfo, out, value) { - return value.Out(out); -} + +Y_DECLARE_OUT_SPEC(inline, NKikimr::NPQ::TSourceIdInfo, out, value) { + return value.Out(out); +} diff --git a/ydb/core/persqueue/sourceid_ut.cpp b/ydb/core/persqueue/sourceid_ut.cpp index 0764128345e..46e03b2d3f2 100644 --- a/ydb/core/persqueue/sourceid_ut.cpp +++ b/ydb/core/persqueue/sourceid_ut.cpp @@ -1,330 +1,330 @@ #include "sourceid.h" - + #include <ydb/core/keyvalue/keyvalue_events.h> #include <ydb/core/persqueue/key.h> - + #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr { -namespace NPQ { +namespace NKikimr { +namespace NPQ { + +Y_UNIT_TEST_SUITE(TSourceIdTests) { + inline static TString TestSourceId(ui64 idx = 0) { + return TStringBuilder() << "testSourceId" << idx; + } -Y_UNIT_TEST_SUITE(TSourceIdTests) { - inline static TString TestSourceId(ui64 idx = 0) { - return TStringBuilder() << "testSourceId" << idx; - } + inline static TStringBuf TestOwner(TStringBuf sourceId) { + UNIT_ASSERT(sourceId.SkipPrefix("test")); + return sourceId; + } - inline static TStringBuf TestOwner(TStringBuf sourceId) { - UNIT_ASSERT(sourceId.SkipPrefix("test")); - return sourceId; - } + static constexpr ui32 TestPartition = 1; - static constexpr ui32 TestPartition = 1; - Y_UNIT_TEST(SourceIdWriterAddMessage) { - TSourceIdWriter writer(ESourceIdFormat::Raw); - - const auto sourceId = TestSourceId(1); - const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); - - writer.RegisterSourceId(sourceId, sourceIdInfo); - UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); - - { - auto it = writer.GetSourceIdsToWrite().find(sourceId); - UNIT_ASSERT_UNEQUAL(it, writer.GetSourceIdsToWrite().end()); - UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); - } - - const auto anotherSourceId = TestSourceId(2); - const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); - UNIT_ASSERT_VALUES_UNEQUAL(sourceIdInfo, anotherSourceIdInfo); - - { - auto it = writer.GetSourceIdsToWrite().find(anotherSourceId); - UNIT_ASSERT_EQUAL(it, writer.GetSourceIdsToWrite().end()); - } + TSourceIdWriter writer(ESourceIdFormat::Raw); + + const auto sourceId = TestSourceId(1); + const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); + + writer.RegisterSourceId(sourceId, sourceIdInfo); + UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); + + { + auto it = writer.GetSourceIdsToWrite().find(sourceId); + UNIT_ASSERT_UNEQUAL(it, writer.GetSourceIdsToWrite().end()); + UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); + } + + const auto anotherSourceId = TestSourceId(2); + const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); + UNIT_ASSERT_VALUES_UNEQUAL(sourceIdInfo, anotherSourceIdInfo); + + { + auto it = writer.GetSourceIdsToWrite().find(anotherSourceId); + UNIT_ASSERT_EQUAL(it, writer.GetSourceIdsToWrite().end()); + } } Y_UNIT_TEST(SourceIdWriterClean) { - TSourceIdWriter writer(ESourceIdFormat::Raw); - - writer.RegisterSourceId(TestSourceId(), 1, 10, TInstant::Seconds(100)); - UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); - - writer.Clear(); - UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 0); + TSourceIdWriter writer(ESourceIdFormat::Raw); + + writer.RegisterSourceId(TestSourceId(), 1, 10, TInstant::Seconds(100)); + UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); + + writer.Clear(); + UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 0); } Y_UNIT_TEST(SourceIdWriterFormCommand) { - TSourceIdWriter writer(ESourceIdFormat::Raw); - auto actualRequest = MakeHolder<TEvKeyValue::TEvRequest>(); - auto expectedRequest = MakeHolder<TEvKeyValue::TEvRequest>(); - - const auto sourceId = TestSourceId(1); - const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); - writer.RegisterSourceId(sourceId, sourceIdInfo); - UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); - { - TKeyPrefix key(TKeyPrefix::TypeInfo, TestPartition, TKeyPrefix::MarkSourceId); - TBuffer data; - - TSourceIdWriter::FillKeyAndData(ESourceIdFormat::Raw, sourceId, sourceIdInfo, key, data); - auto write = expectedRequest.Get()->Record.AddCmdWrite(); - write->SetKey(key.Data(), key.Size()); - write->SetValue(data.Data(), data.Size()); - write->SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); - - writer.FillRequest(actualRequest.Get(), TestPartition); - UNIT_ASSERT_VALUES_EQUAL(actualRequest.Get()->ToString(), expectedRequest.Get()->ToString()); - } - - const auto anotherSourceId = TestSourceId(2); - const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); - writer.RegisterSourceId(anotherSourceId, anotherSourceIdInfo); - UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 2); - { - TKeyPrefix key(TKeyPrefix::TypeInfo, TestPartition + 1, TKeyPrefix::MarkSourceId); - TBuffer data; - - for (const auto& [sourceId, sourceIdInfo] : writer.GetSourceIdsToWrite()) { - TSourceIdWriter::FillKeyAndData(ESourceIdFormat::Raw, sourceId, sourceIdInfo, key, data); - auto write = expectedRequest.Get()->Record.AddCmdWrite(); - write->SetKey(key.Data(), key.Size()); - write->SetValue(data.Data(), data.Size()); - write->SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); - } - - writer.FillRequest(actualRequest.Get(), TestPartition + 1); - UNIT_ASSERT_VALUES_EQUAL(actualRequest.Get()->ToString(), expectedRequest.Get()->ToString()); - } + TSourceIdWriter writer(ESourceIdFormat::Raw); + auto actualRequest = MakeHolder<TEvKeyValue::TEvRequest>(); + auto expectedRequest = MakeHolder<TEvKeyValue::TEvRequest>(); + + const auto sourceId = TestSourceId(1); + const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); + writer.RegisterSourceId(sourceId, sourceIdInfo); + UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 1); + { + TKeyPrefix key(TKeyPrefix::TypeInfo, TestPartition, TKeyPrefix::MarkSourceId); + TBuffer data; + + TSourceIdWriter::FillKeyAndData(ESourceIdFormat::Raw, sourceId, sourceIdInfo, key, data); + auto write = expectedRequest.Get()->Record.AddCmdWrite(); + write->SetKey(key.Data(), key.Size()); + write->SetValue(data.Data(), data.Size()); + write->SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); + + writer.FillRequest(actualRequest.Get(), TestPartition); + UNIT_ASSERT_VALUES_EQUAL(actualRequest.Get()->ToString(), expectedRequest.Get()->ToString()); + } + + const auto anotherSourceId = TestSourceId(2); + const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); + writer.RegisterSourceId(anotherSourceId, anotherSourceIdInfo); + UNIT_ASSERT_VALUES_EQUAL(writer.GetSourceIdsToWrite().size(), 2); + { + TKeyPrefix key(TKeyPrefix::TypeInfo, TestPartition + 1, TKeyPrefix::MarkSourceId); + TBuffer data; + + for (const auto& [sourceId, sourceIdInfo] : writer.GetSourceIdsToWrite()) { + TSourceIdWriter::FillKeyAndData(ESourceIdFormat::Raw, sourceId, sourceIdInfo, key, data); + auto write = expectedRequest.Get()->Record.AddCmdWrite(); + write->SetKey(key.Data(), key.Size()); + write->SetValue(data.Data(), data.Size()); + write->SetStorageChannel(NKikimrClient::TKeyValueRequest::INLINE); + } + + writer.FillRequest(actualRequest.Get(), TestPartition + 1); + UNIT_ASSERT_VALUES_EQUAL(actualRequest.Get()->ToString(), expectedRequest.Get()->ToString()); + } } Y_UNIT_TEST(SourceIdStorageAdd) { - TSourceIdStorage storage; - - const auto sourceId = TestSourceId(1); - const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); - const auto anotherSourceId = TestSourceId(2); - const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); - - storage.RegisterSourceId(sourceId, sourceIdInfo); - UNIT_ASSERT_VALUES_EQUAL(storage.GetInMemorySourceIds().size(), 1); - { - auto it = storage.GetInMemorySourceIds().find(sourceId); - UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); - UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); - } - { - auto it = storage.GetInMemorySourceIds().find(anotherSourceId); - UNIT_ASSERT_EQUAL(it, storage.GetInMemorySourceIds().end()); - } - - storage.RegisterSourceId(anotherSourceId, anotherSourceIdInfo); - storage.RegisterSourceId(sourceId, anotherSourceIdInfo); - UNIT_ASSERT_VALUES_EQUAL(storage.GetInMemorySourceIds().size(), 2); - { - auto it = storage.GetInMemorySourceIds().find(sourceId); - UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); - UNIT_ASSERT_VALUES_EQUAL(it->second, anotherSourceIdInfo); - } + TSourceIdStorage storage; + + const auto sourceId = TestSourceId(1); + const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); + const auto anotherSourceId = TestSourceId(2); + const auto anotherSourceIdInfo = TSourceIdInfo(2, 20, TInstant::Seconds(200)); + + storage.RegisterSourceId(sourceId, sourceIdInfo); + UNIT_ASSERT_VALUES_EQUAL(storage.GetInMemorySourceIds().size(), 1); + { + auto it = storage.GetInMemorySourceIds().find(sourceId); + UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); + UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); + } + { + auto it = storage.GetInMemorySourceIds().find(anotherSourceId); + UNIT_ASSERT_EQUAL(it, storage.GetInMemorySourceIds().end()); + } + + storage.RegisterSourceId(anotherSourceId, anotherSourceIdInfo); + storage.RegisterSourceId(sourceId, anotherSourceIdInfo); + UNIT_ASSERT_VALUES_EQUAL(storage.GetInMemorySourceIds().size(), 2); + { + auto it = storage.GetInMemorySourceIds().find(sourceId); + UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); + UNIT_ASSERT_VALUES_EQUAL(it->second, anotherSourceIdInfo); + } + } + + void SourceIdStorageParseAndAdd(TKeyPrefix::EMark mark, ESourceIdFormat format) { + const auto sourceId = TestSourceId(); + const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); + + TKeyPrefix ikey(TKeyPrefix::TypeInfo, TestPartition, mark); + TBuffer idata; + TSourceIdWriter::FillKeyAndData(format, sourceId, sourceIdInfo, ikey, idata); + + TString key; + ikey.AsString(key); + TString data; + idata.AsString(data); + + TSourceIdStorage storage; + storage.LoadSourceIdInfo(key, data, TInstant()); + + auto it = storage.GetInMemorySourceIds().find(sourceId); + UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); + UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); } - void SourceIdStorageParseAndAdd(TKeyPrefix::EMark mark, ESourceIdFormat format) { - const auto sourceId = TestSourceId(); - const auto sourceIdInfo = TSourceIdInfo(1, 10, TInstant::Seconds(100)); - - TKeyPrefix ikey(TKeyPrefix::TypeInfo, TestPartition, mark); - TBuffer idata; - TSourceIdWriter::FillKeyAndData(format, sourceId, sourceIdInfo, ikey, idata); - - TString key; - ikey.AsString(key); - TString data; - idata.AsString(data); - - TSourceIdStorage storage; - storage.LoadSourceIdInfo(key, data, TInstant()); - - auto it = storage.GetInMemorySourceIds().find(sourceId); - UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); - UNIT_ASSERT_VALUES_EQUAL(it->second, sourceIdInfo); + Y_UNIT_TEST(SourceIdStorageParseAndAdd) { + SourceIdStorageParseAndAdd(TKeyPrefix::MarkSourceId, ESourceIdFormat::Raw); + } + + Y_UNIT_TEST(ProtoSourceIdStorageParseAndAdd) { + SourceIdStorageParseAndAdd(TKeyPrefix::MarkProtoSourceId, ESourceIdFormat::Proto); } - Y_UNIT_TEST(SourceIdStorageParseAndAdd) { - SourceIdStorageParseAndAdd(TKeyPrefix::MarkSourceId, ESourceIdFormat::Raw); - } - - Y_UNIT_TEST(ProtoSourceIdStorageParseAndAdd) { - SourceIdStorageParseAndAdd(TKeyPrefix::MarkProtoSourceId, ESourceIdFormat::Proto); - } - Y_UNIT_TEST(SourceIdStorageMinDS) { - const auto now = TInstant::Now(); - TSourceIdStorage storage; - - const auto sourceId = TestSourceId(1); - storage.RegisterSourceId(sourceId, 1, 10, TInstant::Seconds(100)); - { - auto ds = storage.MinAvailableTimestamp(now); - UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(100)); - } - - const auto anotherSourceId = TestSourceId(2); - storage.RegisterSourceId(anotherSourceId, 2, 20, TInstant::Seconds(200)); - { - auto ds = storage.MinAvailableTimestamp(now); - UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(100)); - } - - storage.RegisterSourceId(sourceId, 3, 30, TInstant::Seconds(300)); - { - auto ds = storage.MinAvailableTimestamp(now); - UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(200)); - } + const auto now = TInstant::Now(); + TSourceIdStorage storage; + + const auto sourceId = TestSourceId(1); + storage.RegisterSourceId(sourceId, 1, 10, TInstant::Seconds(100)); + { + auto ds = storage.MinAvailableTimestamp(now); + UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(100)); + } + + const auto anotherSourceId = TestSourceId(2); + storage.RegisterSourceId(anotherSourceId, 2, 20, TInstant::Seconds(200)); + { + auto ds = storage.MinAvailableTimestamp(now); + UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(100)); + } + + storage.RegisterSourceId(sourceId, 3, 30, TInstant::Seconds(300)); + { + auto ds = storage.MinAvailableTimestamp(now); + UNIT_ASSERT_VALUES_EQUAL(ds, TInstant::Seconds(200)); + } } Y_UNIT_TEST(SourceIdStorageTestClean) { - TSourceIdStorage storage; - for (ui64 i = 1; i <= 10000; ++i) { - storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); - } - - NKikimrPQ::TPartitionConfig config; - config.SetSourceIdLifetimeSeconds(TDuration::Hours(1).Seconds()); - - // sources are dropped before startOffset = 20 - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 20, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 19); // first 19 sources are dropped - } - - // expired sources are dropped - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 341); // another 341 (360 - 19) sources are dropped - } - - // move to the past - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, false); // nothing to drop (everything is dropped) - } - - // move to the future - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(3), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 360); // more 360 sources are dropped - } + TSourceIdStorage storage; + for (ui64 i = 1; i <= 10000; ++i) { + storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); + } + + NKikimrPQ::TPartitionConfig config; + config.SetSourceIdLifetimeSeconds(TDuration::Hours(1).Seconds()); + + // sources are dropped before startOffset = 20 + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 20, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 19); // first 19 sources are dropped + } + + // expired sources are dropped + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 341); // another 341 (360 - 19) sources are dropped + } + + // move to the past + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, false); // nothing to drop (everything is dropped) + } + + // move to the future + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(3), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 360); // more 360 sources are dropped + } } Y_UNIT_TEST(SourceIdStorageDeleteByMaxCount) { - TSourceIdStorage storage; - for (ui64 i = 1; i <= 10000; ++i) { - storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); - } - - NKikimrPQ::TPartitionConfig config; - - config.SetSourceIdMaxCounts(10000); - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, false); - } - - config.SetSourceIdMaxCounts(9900); // decrease by 100 - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 100); // 100 sources are dropped - } - { - auto it = storage.GetInMemorySourceIds().find(TestSourceId(100)); // 100th source is dropped - UNIT_ASSERT_EQUAL(it, storage.GetInMemorySourceIds().end()); - } - { - auto it = storage.GetInMemorySourceIds().find(TestSourceId(101)); // 101th source is alive - UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); - } + TSourceIdStorage storage; + for (ui64 i = 1; i <= 10000; ++i) { + storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); + } + + NKikimrPQ::TPartitionConfig config; + + config.SetSourceIdMaxCounts(10000); + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, false); + } + + config.SetSourceIdMaxCounts(9900); // decrease by 100 + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 100); // 100 sources are dropped + } + { + auto it = storage.GetInMemorySourceIds().find(TestSourceId(100)); // 100th source is dropped + UNIT_ASSERT_EQUAL(it, storage.GetInMemorySourceIds().end()); + } + { + auto it = storage.GetInMemorySourceIds().find(TestSourceId(101)); // 101th source is alive + UNIT_ASSERT_UNEQUAL(it, storage.GetInMemorySourceIds().end()); + } } Y_UNIT_TEST(SourceIdStorageComplexDelete) { - TSourceIdStorage storage; - for (ui64 i = 1; i <= 10000 + 1; ++i) { // add 10000 + one extra sources - storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); - } - - NKikimrPQ::TPartitionConfig config; - - config.SetSourceIdLifetimeSeconds(TDuration::Hours(1).Seconds()); - config.SetSourceIdMaxCounts(10000); // limit to 10000 - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 360); // first 360 sources are dropped - } - - config.SetSourceIdLifetimeSeconds((TDuration::Hours(1) - TDuration::Minutes(1)).Seconds()); - config.SetSourceIdMaxCounts(10000 - 360); - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 6); // another 6 sources are dropped by retention - } - - config.SetSourceIdMaxCounts(10000 - 370); - { - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 5); // more 5 sources are dropped - } + TSourceIdStorage storage; + for (ui64 i = 1; i <= 10000 + 1; ++i) { // add 10000 + one extra sources + storage.RegisterSourceId(TestSourceId(i), i, i, TInstant::Seconds(10 * i)); + } + + NKikimrPQ::TPartitionConfig config; + + config.SetSourceIdLifetimeSeconds(TDuration::Hours(1).Seconds()); + config.SetSourceIdMaxCounts(10000); // limit to 10000 + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 360); // first 360 sources are dropped + } + + config.SetSourceIdLifetimeSeconds((TDuration::Hours(1) - TDuration::Minutes(1)).Seconds()); + config.SetSourceIdMaxCounts(10000 - 360); + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 6); // another 6 sources are dropped by retention + } + + config.SetSourceIdMaxCounts(10000 - 370); + { + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(2), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2 * 5); // more 5 sources are dropped + } } Y_UNIT_TEST(SourceIdStorageDeleteAndOwnersMark) { - TSourceIdStorage storage; - THashMap<TString, TOwnerInfo> owners; - for (ui64 i = 1; i <= 2; ++i) { // add two sources - const auto sourceId = TestSourceId(i); - const auto owner = TestOwner(sourceId); - - storage.RegisterSourceId(sourceId, i, i, TInstant::Hours(i)); - storage.RegisterSourceIdOwner(sourceId, owner); - owners[owner]; - } - - auto request = MakeHolder<TEvKeyValue::TEvRequest>(); - NKikimrPQ::TPartitionConfig config; - config.SetSourceIdMaxCounts(1); // limit to one - - const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); - UNIT_ASSERT_EQUAL(dropped, true); - UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2); // first source is dropped by retention - - storage.MarkOwnersForDeletedSourceId(owners); - { - auto it = owners.find(TestOwner(TestSourceId(1))); - UNIT_ASSERT_UNEQUAL(it, owners.end()); - UNIT_ASSERT_VALUES_EQUAL(it->second.SourceIdDeleted, true); - } - { - auto it = owners.find(TestOwner(TestSourceId(2))); - UNIT_ASSERT_UNEQUAL(it, owners.end()); - UNIT_ASSERT_VALUES_EQUAL(it->second.SourceIdDeleted, false); - } + TSourceIdStorage storage; + THashMap<TString, TOwnerInfo> owners; + for (ui64 i = 1; i <= 2; ++i) { // add two sources + const auto sourceId = TestSourceId(i); + const auto owner = TestOwner(sourceId); + + storage.RegisterSourceId(sourceId, i, i, TInstant::Hours(i)); + storage.RegisterSourceIdOwner(sourceId, owner); + owners[owner]; + } + + auto request = MakeHolder<TEvKeyValue::TEvRequest>(); + NKikimrPQ::TPartitionConfig config; + config.SetSourceIdMaxCounts(1); // limit to one + + const auto dropped = storage.DropOldSourceIds(request.Get(), TInstant::Hours(1), 10000, TestPartition, config); + UNIT_ASSERT_EQUAL(dropped, true); + UNIT_ASSERT_VALUES_EQUAL(request.Get()->Record.CmdDeleteRangeSize(), 2); // first source is dropped by retention + + storage.MarkOwnersForDeletedSourceId(owners); + { + auto it = owners.find(TestOwner(TestSourceId(1))); + UNIT_ASSERT_UNEQUAL(it, owners.end()); + UNIT_ASSERT_VALUES_EQUAL(it->second.SourceIdDeleted, true); + } + { + auto it = owners.find(TestOwner(TestSourceId(2))); + UNIT_ASSERT_UNEQUAL(it, owners.end()); + UNIT_ASSERT_VALUES_EQUAL(it->second.SourceIdDeleted, false); + } } -} // TSourceIdTests +} // TSourceIdTests -} // NPQ -} // NKikimr +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/writer/source_id_encoding.cpp b/ydb/core/persqueue/writer/source_id_encoding.cpp index fa6264a3bc4..13a37c2c61b 100644 --- a/ydb/core/persqueue/writer/source_id_encoding.cpp +++ b/ydb/core/persqueue/writer/source_id_encoding.cpp @@ -6,84 +6,84 @@ #include <util/string/strip.h> namespace NKikimr { -namespace NPQ { -namespace NSourceIdEncoding { - -struct TTags { - static constexpr char Simple = 0; - static constexpr char Base64 = 1; -}; - -static constexpr TStringBuf Base64Prefix = "base64:"; - +namespace NPQ { +namespace NSourceIdEncoding { + +struct TTags { + static constexpr char Simple = 0; + static constexpr char Base64 = 1; +}; + +static constexpr TStringBuf Base64Prefix = "base64:"; + TString EncodeSimple(const TString& sourceId) { - return TString(1, TTags::Simple) + sourceId; + return TString(1, TTags::Simple) + sourceId; +} + +TString DecodeSimple(const TString& sourceId) { + Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Simple); + return sourceId.substr(1); +} + +TString EncodeBase64(const TString& sourceId) { + const auto size = sourceId.size(); + const auto prefixSize = Base64Prefix.size(); + + if (size == prefixSize || sourceId.find('=') != TString::npos) { + ythrow yexception() << "started from \"" << Base64Prefix << "\" must be valid base64 encoded string, without padding"; + } + + const auto remainder = (size - prefixSize) % 4; + const auto padding = remainder ? TString(4 - remainder, '=') : ""; + const auto effective = sourceId.substr(prefixSize) + padding; + + try { + return TString(1, TTags::Base64) + Base64StrictDecode(effective); + } catch (yexception& e) { + ythrow yexception() << "started from \"" << Base64Prefix << "\" must be valid base64 encoded string: " << e.what(); + } +} + +TString DecodeBase64(const TString& sourceId) { + Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Base64); + return Base64Prefix + StripStringRight(Base64EncodeUrl(sourceId.substr(1)), EqualsStripAdapter(',')); } -TString DecodeSimple(const TString& sourceId) { - Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Simple); - return sourceId.substr(1); -} - -TString EncodeBase64(const TString& sourceId) { - const auto size = sourceId.size(); - const auto prefixSize = Base64Prefix.size(); - - if (size == prefixSize || sourceId.find('=') != TString::npos) { - ythrow yexception() << "started from \"" << Base64Prefix << "\" must be valid base64 encoded string, without padding"; - } - - const auto remainder = (size - prefixSize) % 4; - const auto padding = remainder ? TString(4 - remainder, '=') : ""; - const auto effective = sourceId.substr(prefixSize) + padding; - - try { - return TString(1, TTags::Base64) + Base64StrictDecode(effective); - } catch (yexception& e) { - ythrow yexception() << "started from \"" << Base64Prefix << "\" must be valid base64 encoded string: " << e.what(); - } -} - -TString DecodeBase64(const TString& sourceId) { - Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Base64); - return Base64Prefix + StripStringRight(Base64EncodeUrl(sourceId.substr(1)), EqualsStripAdapter(',')); -} - TString Encode(const TString& sourceId) { - if (sourceId.StartsWith(Base64Prefix)) { - return EncodeBase64(sourceId); + if (sourceId.StartsWith(Base64Prefix)) { + return EncodeBase64(sourceId); } else { - return EncodeSimple(sourceId); + return EncodeSimple(sourceId); } } -TString Decode(const TString& sourceId) { - Y_VERIFY(!sourceId.empty()); - - switch (sourceId[0]) { - case TTags::Simple: - return DecodeSimple(sourceId); - case TTags::Base64: - return DecodeBase64(sourceId); - default: - return sourceId; +TString Decode(const TString& sourceId) { + Y_VERIFY(!sourceId.empty()); + + switch (sourceId[0]) { + case TTags::Simple: + return DecodeSimple(sourceId); + case TTags::Base64: + return DecodeBase64(sourceId); + default: + return sourceId; } } -bool IsValidEncoded(const TString& sourceId) { - if (sourceId.empty()) { - return false; - } - - switch (sourceId[0]) { - case TTags::Simple: - case TTags::Base64: - return true; - default: - return false; - } +bool IsValidEncoded(const TString& sourceId) { + if (sourceId.empty()) { + return false; + } + + switch (sourceId[0]) { + case TTags::Simple: + case TTags::Base64: + return true; + default: + return false; + } } -} // NSourceIdEncoding -} // NPQ -} // NKikimr +} // NSourceIdEncoding +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/writer/source_id_encoding.h b/ydb/core/persqueue/writer/source_id_encoding.h index 999785f7940..6472d8b4ac9 100644 --- a/ydb/core/persqueue/writer/source_id_encoding.h +++ b/ydb/core/persqueue/writer/source_id_encoding.h @@ -3,8 +3,8 @@ #include <util/generic/fwd.h> namespace NKikimr { -namespace NPQ { -namespace NSourceIdEncoding { +namespace NPQ { +namespace NSourceIdEncoding { TString EncodeSimple(const TString& sourceId); TString Encode(const TString& sourceId); @@ -13,6 +13,6 @@ TString Decode(const TString& encodedSourceId); bool IsValidEncoded(const TString& encodedSourceId); -} // NSourceIdEncoding -} // NPQ -} // NKikimr +} // NSourceIdEncoding +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index c96e09e2c56..12a88372fbe 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -1,457 +1,457 @@ -#include "source_id_encoding.h" -#include "writer.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - +#include "source_id_encoding.h" +#include "writer.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/persqueue/events/global.h> #include <ydb/public/lib/base/msgbus_status.h> - -#include <util/generic/deque.h> -#include <util/generic/map.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NPQ { - -TString TEvPartitionWriter::TEvInitResult::ToString() const { - auto out = TStringBuilder() << ToStringHeader() << " {"; - - if (IsSuccess()) { - const auto& result = GetResult(); - out << " OwnerCookie: " << result.OwnerCookie - << " SourceIdInfo: " << result.SourceIdInfo.ShortDebugString(); - } else { - const auto& error = GetError(); - out << " Reason: " << error.Reason - << " Response: " << error.Response.ShortDebugString(); - } - - out << " }"; - return out; -} - -TString TEvPartitionWriter::TEvWriteAccepted::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Cookie: " << Cookie - << " }"; -} - -class TPartitionWriter: public TActorBootstrapped<TPartitionWriter> { - static void FillHeader(NKikimrClient::TPersQueuePartitionRequest& request, - ui32 partitionId, const TActorId& pipeClient) - { - request.SetPartition(partitionId); - ActorIdToProto(pipeClient, request.MutablePipeClient()); - } - - static void FillHeader(NKikimrClient::TPersQueuePartitionRequest& request, - ui32 partitionId, const TActorId& pipeClient, const TString& ownerCookie, ui64 cookie) - { - FillHeader(request, partitionId, pipeClient); - request.SetOwnerCookie(ownerCookie); - request.SetCookie(cookie); - } - - template <typename... Args> - static THolder<TEvPersQueue::TEvRequest> MakeRequest(Args&&... args) { - auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); - FillHeader(*ev->Record.MutablePartitionRequest(), std::forward<Args>(args)...); - - return ev; - } - - static bool BasicCheck(const NKikimrClient::TResponse& response, TString& error, bool mustHaveResponse = true) { - if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - error = TStringBuilder() << "Status is not ok" - << ": status# " << static_cast<ui32>(response.GetStatus()); - return false; - } - - if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) { - error = TStringBuilder() << "Error code is not ok" - << ": code# " << static_cast<ui32>(response.GetErrorCode()); - return false; - } - - if (mustHaveResponse && !response.HasPartitionResponse()) { - error = "Absent partition response"; - return false; - } - - return true; - } - - void InitResult(const TString& reason, NKikimrClient::TResponse&& response) { - Send(Client, new TEvPartitionWriter::TEvInitResult(reason, std::move(response))); - Become(&TThis::StateZombie); - } - - void InitResult(const TString& ownerCookie, const TEvPartitionWriter::TEvInitResult::TSourceIdInfo& sourceIdInfo) { - Send(Client, new TEvPartitionWriter::TEvInitResult(ownerCookie, sourceIdInfo)); - } - - void WriteResult(const TString& reason, NKikimrClient::TResponse&& response) { - Send(Client, new TEvPartitionWriter::TEvWriteResponse(reason, std::move(response))); - Become(&TThis::StateZombie); - } - - void WriteResult(NKikimrClient::TResponse&& response) { - Send(Client, new TEvPartitionWriter::TEvWriteResponse(std::move(response))); - PendingWrite.pop_front(); - } - - void WriteAccepted(ui64 cookie) { - Send(Client, new TEvPartitionWriter::TEvWriteAccepted(cookie)); - } - - void Disconnected() { - Send(Client, new TEvPartitionWriter::TEvDisconnected()); - Become(&TThis::StateZombie); - } - - /// GetOwnership - - void GetOwnership() { - auto ev = MakeRequest(PartitionId, PipeClient); - - auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdGetOwnership(); - cmd.SetOwner(SourceId); - cmd.SetForce(true); - - NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); - Become(&TThis::StateGetOwnership); - } - - STATEFN(StateGetOwnership) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPersQueue::TEvResponse, HandleOwnership); - hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleOwnership(TEvPersQueue::TEvResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - - TString error; - if (!BasicCheck(record, error)) { - return InitResult(error, std::move(record)); - } - - const auto& response = record.GetPartitionResponse(); - if (!response.HasCmdGetOwnershipResult()) { - return InitResult("Absent Ownership result", std::move(record)); - } - - OwnerCookie = response.GetCmdGetOwnershipResult().GetOwnerCookie(); - GetMaxSeqNo(); - } - - /// GetMaxSeqNo - - void GetMaxSeqNo() { - auto ev = MakeRequest(PartitionId, PipeClient); - - auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdGetMaxSeqNo(); - cmd.AddSourceId(NSourceIdEncoding::EncodeSimple(SourceId)); - - NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); - Become(&TThis::StateGetMaxSeqNo); - } - - STATEFN(StateGetMaxSeqNo) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPersQueue::TEvResponse, HandleMaxSeqNo); - hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleMaxSeqNo(TEvPersQueue::TEvResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - - TString error; - if (!BasicCheck(record, error)) { - return InitResult(error, std::move(record)); - } - - const auto& response = record.GetPartitionResponse(); - if (!response.HasCmdGetMaxSeqNoResult()) { - return InitResult("Absent MaxSeqNo result", std::move(record)); - } - - const auto& result = response.GetCmdGetMaxSeqNoResult(); - if (result.SourceIdInfoSize() < 1) { - return InitResult("Empty source id info", std::move(record)); - } - - const auto& sourceIdInfo = result.GetSourceIdInfo(0); - if (Opts.CheckState) { - switch (sourceIdInfo.GetState()) { - case NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED: - Registered = true; - break; - case NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION: - if (Opts.AutoRegister) { - return RegisterMessageGroup(); - } else { - return InitResult("Source is not registered", std::move(record)); - } - default: - return InitResult("Unknown source state", std::move(record)); - } - } - - InitResult(OwnerCookie, sourceIdInfo); - Become(&TThis::StateWork); - - if (Pending) { - ReserveBytes(); - } - } - - /// RegisterMessageGroup - - void RegisterMessageGroup() { - if (Registered) { - Y_VERIFY_DEBUG(false); - return InitResult("Already registered", NKikimrClient::TResponse()); - } - - auto ev = MakeRequest(PartitionId, PipeClient); - - auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdRegisterMessageGroup(); - cmd.SetId(NSourceIdEncoding::EncodeSimple(SourceId)); - // TODO cmd.SetPartitionKeyRange() - cmd.SetAfterSplit(true); - - NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); - Become(&TThis::StateRegisterMessageGroup); - } - - STATEFN(StateRegisterMessageGroup) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPersQueue::TEvResponse, HandleRegisterMessageGroup); - hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleRegisterMessageGroup(TEvPersQueue::TEvResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - - TString error; - if (!BasicCheck(record, error, false)) { - return InitResult(error, std::move(record)); - } - - Registered = true; - GetMaxSeqNo(); - } - - /// Work - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPartitionWriter::TEvWriteRequest, Handle); - hFunc(TEvPersQueue::TEvResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HoldPending(TEvPartitionWriter::TEvWriteRequest::TPtr& ev) { - auto& record = ev->Get()->Record; - const auto cookie = record.GetPartitionRequest().GetCookie(); - - Y_VERIFY(Pending.empty() || Pending.rbegin()->first < cookie); - Y_VERIFY(PendingReserve.empty() || PendingReserve.rbegin()->first < cookie); - Y_VERIFY(PendingWrite.empty() || PendingWrite.back() < cookie); - - Pending.emplace(cookie, std::move(ev->Get()->Record)); - } - - void Handle(TEvPartitionWriter::TEvWriteRequest::TPtr& ev) { - HoldPending(ev); - ReserveBytes(); - } - - void ReserveBytes() { - while (Pending) { - auto it = Pending.begin(); - - FillHeader(*it->second.MutablePartitionRequest(), PartitionId, PipeClient, OwnerCookie, it->first); - auto ev = MakeRequest(PartitionId, PipeClient, OwnerCookie, it->first); - - auto& request = *ev->Record.MutablePartitionRequest(); - request.SetMessageNo(MessageNo++); - - auto& cmd = *request.MutableCmdReserveBytes(); - cmd.SetSize(it->second.ByteSize()); - cmd.SetLastRequest(false); - - NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); - - PendingReserve.emplace(it->first, std::move(it->second)); - Pending.erase(it); - } - } - - void Write(ui64 cookie) { - Y_VERIFY(!PendingReserve.empty()); - auto it = PendingReserve.begin(); - - Y_VERIFY(it->first == cookie); - Y_VERIFY(PendingWrite.empty() || PendingWrite.back() < cookie); - - auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); - ev->Record = std::move(it->second); - - auto& request = *ev->Record.MutablePartitionRequest(); - request.SetMessageNo(MessageNo++); - - NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); - - PendingWrite.emplace_back(cookie); - PendingReserve.erase(it); - } - - void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - - TString error; - if (!BasicCheck(record, error)) { - return WriteResult(error, std::move(record)); - } - - const auto& response = record.GetPartitionResponse(); - if (!response.CmdWriteResultSize()) { - if (PendingReserve.empty()) { - return WriteResult("Unexpected ReserveBytes response", std::move(record)); - } - - const auto cookie = PendingReserve.begin()->first; - if (cookie != response.GetCookie()) { - error = TStringBuilder() << "Unexpected cookie at ReserveBytes" - << ": expected# " << cookie - << ", got# " << response.GetCookie(); - return WriteResult(error, std::move(record)); - } - - WriteAccepted(cookie); - Write(cookie); - } else { - if (PendingWrite.empty()) { - return WriteResult("Unexpected Write response", std::move(record)); - } - - const auto cookie = PendingWrite.front(); - if (cookie != response.GetCookie()) { - error = TStringBuilder() << "Unexpected cookie at Write" - << ": expected# " << cookie - << ", got# " << response.GetCookie(); - return WriteResult(error, std::move(record)); - } - - WriteResult(std::move(record)); - } - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - if (ev->Get()->TabletId == TabletId && ev->Get()->Status != NKikimrProto::OK) { - Disconnected(); - } - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { - if (ev->Get()->TabletId == TabletId) { - Disconnected(); - } - } - - void PassAway() override { - if (PipeClient) { - NTabletPipe::CloseAndForgetClient(SelfId(), PipeClient); - } - - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::PQ_PARTITION_WRITER_ACTOR; - } - - explicit TPartitionWriter( - const TActorId& client, - ui64 tabletId, - ui32 partitionId, - const TString& sourceId, - const TPartitionWriterOpts& opts) - : Client(client) - , TabletId(tabletId) - , PartitionId(partitionId) - , SourceId(sourceId) - , Opts(opts) - { - } - - void Bootstrap() { - NTabletPipe::TClientConfig config; - config.RetryPolicy = { - .RetryLimitCount = 6, - .MinRetryTime = TDuration::MilliSeconds(10), - .MaxRetryTime = TDuration::MilliSeconds(100), - .BackoffMultiplier = 2, - .DoFirstRetryInstantly = true - }; - - PipeClient = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), TabletId, config)); - - GetOwnership(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - - STATEFN(StateZombie) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Client; - const ui64 TabletId; - const ui32 PartitionId; - const TString SourceId; - const TPartitionWriterOpts Opts; - - TActorId PipeClient; - TString OwnerCookie; - bool Registered = false; - ui64 MessageNo = 0; - - TMap<ui64, NKikimrClient::TPersQueueRequest> Pending; - TMap<ui64, NKikimrClient::TPersQueueRequest> PendingReserve; - TDeque<ui64> PendingWrite; - -}; // TPartitionWriter - -IActor* CreatePartitionWriter(const TActorId& client, ui64 tabletId, ui32 partitionId, const TString& sourceId, const TPartitionWriterOpts& opts) { - return new TPartitionWriter(client, tabletId, partitionId, sourceId, opts); -} - -} // NPQ -} // NKikimr + +#include <util/generic/deque.h> +#include <util/generic/map.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NPQ { + +TString TEvPartitionWriter::TEvInitResult::ToString() const { + auto out = TStringBuilder() << ToStringHeader() << " {"; + + if (IsSuccess()) { + const auto& result = GetResult(); + out << " OwnerCookie: " << result.OwnerCookie + << " SourceIdInfo: " << result.SourceIdInfo.ShortDebugString(); + } else { + const auto& error = GetError(); + out << " Reason: " << error.Reason + << " Response: " << error.Response.ShortDebugString(); + } + + out << " }"; + return out; +} + +TString TEvPartitionWriter::TEvWriteAccepted::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Cookie: " << Cookie + << " }"; +} + +class TPartitionWriter: public TActorBootstrapped<TPartitionWriter> { + static void FillHeader(NKikimrClient::TPersQueuePartitionRequest& request, + ui32 partitionId, const TActorId& pipeClient) + { + request.SetPartition(partitionId); + ActorIdToProto(pipeClient, request.MutablePipeClient()); + } + + static void FillHeader(NKikimrClient::TPersQueuePartitionRequest& request, + ui32 partitionId, const TActorId& pipeClient, const TString& ownerCookie, ui64 cookie) + { + FillHeader(request, partitionId, pipeClient); + request.SetOwnerCookie(ownerCookie); + request.SetCookie(cookie); + } + + template <typename... Args> + static THolder<TEvPersQueue::TEvRequest> MakeRequest(Args&&... args) { + auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); + FillHeader(*ev->Record.MutablePartitionRequest(), std::forward<Args>(args)...); + + return ev; + } + + static bool BasicCheck(const NKikimrClient::TResponse& response, TString& error, bool mustHaveResponse = true) { + if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) { + error = TStringBuilder() << "Status is not ok" + << ": status# " << static_cast<ui32>(response.GetStatus()); + return false; + } + + if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) { + error = TStringBuilder() << "Error code is not ok" + << ": code# " << static_cast<ui32>(response.GetErrorCode()); + return false; + } + + if (mustHaveResponse && !response.HasPartitionResponse()) { + error = "Absent partition response"; + return false; + } + + return true; + } + + void InitResult(const TString& reason, NKikimrClient::TResponse&& response) { + Send(Client, new TEvPartitionWriter::TEvInitResult(reason, std::move(response))); + Become(&TThis::StateZombie); + } + + void InitResult(const TString& ownerCookie, const TEvPartitionWriter::TEvInitResult::TSourceIdInfo& sourceIdInfo) { + Send(Client, new TEvPartitionWriter::TEvInitResult(ownerCookie, sourceIdInfo)); + } + + void WriteResult(const TString& reason, NKikimrClient::TResponse&& response) { + Send(Client, new TEvPartitionWriter::TEvWriteResponse(reason, std::move(response))); + Become(&TThis::StateZombie); + } + + void WriteResult(NKikimrClient::TResponse&& response) { + Send(Client, new TEvPartitionWriter::TEvWriteResponse(std::move(response))); + PendingWrite.pop_front(); + } + + void WriteAccepted(ui64 cookie) { + Send(Client, new TEvPartitionWriter::TEvWriteAccepted(cookie)); + } + + void Disconnected() { + Send(Client, new TEvPartitionWriter::TEvDisconnected()); + Become(&TThis::StateZombie); + } + + /// GetOwnership + + void GetOwnership() { + auto ev = MakeRequest(PartitionId, PipeClient); + + auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdGetOwnership(); + cmd.SetOwner(SourceId); + cmd.SetForce(true); + + NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); + Become(&TThis::StateGetOwnership); + } + + STATEFN(StateGetOwnership) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPersQueue::TEvResponse, HandleOwnership); + hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleOwnership(TEvPersQueue::TEvResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + + TString error; + if (!BasicCheck(record, error)) { + return InitResult(error, std::move(record)); + } + + const auto& response = record.GetPartitionResponse(); + if (!response.HasCmdGetOwnershipResult()) { + return InitResult("Absent Ownership result", std::move(record)); + } + + OwnerCookie = response.GetCmdGetOwnershipResult().GetOwnerCookie(); + GetMaxSeqNo(); + } + + /// GetMaxSeqNo + + void GetMaxSeqNo() { + auto ev = MakeRequest(PartitionId, PipeClient); + + auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdGetMaxSeqNo(); + cmd.AddSourceId(NSourceIdEncoding::EncodeSimple(SourceId)); + + NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); + Become(&TThis::StateGetMaxSeqNo); + } + + STATEFN(StateGetMaxSeqNo) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPersQueue::TEvResponse, HandleMaxSeqNo); + hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleMaxSeqNo(TEvPersQueue::TEvResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + + TString error; + if (!BasicCheck(record, error)) { + return InitResult(error, std::move(record)); + } + + const auto& response = record.GetPartitionResponse(); + if (!response.HasCmdGetMaxSeqNoResult()) { + return InitResult("Absent MaxSeqNo result", std::move(record)); + } + + const auto& result = response.GetCmdGetMaxSeqNoResult(); + if (result.SourceIdInfoSize() < 1) { + return InitResult("Empty source id info", std::move(record)); + } + + const auto& sourceIdInfo = result.GetSourceIdInfo(0); + if (Opts.CheckState) { + switch (sourceIdInfo.GetState()) { + case NKikimrPQ::TMessageGroupInfo::STATE_REGISTERED: + Registered = true; + break; + case NKikimrPQ::TMessageGroupInfo::STATE_PENDING_REGISTRATION: + if (Opts.AutoRegister) { + return RegisterMessageGroup(); + } else { + return InitResult("Source is not registered", std::move(record)); + } + default: + return InitResult("Unknown source state", std::move(record)); + } + } + + InitResult(OwnerCookie, sourceIdInfo); + Become(&TThis::StateWork); + + if (Pending) { + ReserveBytes(); + } + } + + /// RegisterMessageGroup + + void RegisterMessageGroup() { + if (Registered) { + Y_VERIFY_DEBUG(false); + return InitResult("Already registered", NKikimrClient::TResponse()); + } + + auto ev = MakeRequest(PartitionId, PipeClient); + + auto& cmd = *ev->Record.MutablePartitionRequest()->MutableCmdRegisterMessageGroup(); + cmd.SetId(NSourceIdEncoding::EncodeSimple(SourceId)); + // TODO cmd.SetPartitionKeyRange() + cmd.SetAfterSplit(true); + + NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); + Become(&TThis::StateRegisterMessageGroup); + } + + STATEFN(StateRegisterMessageGroup) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPersQueue::TEvResponse, HandleRegisterMessageGroup); + hFunc(TEvPartitionWriter::TEvWriteRequest, HoldPending); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleRegisterMessageGroup(TEvPersQueue::TEvResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + + TString error; + if (!BasicCheck(record, error, false)) { + return InitResult(error, std::move(record)); + } + + Registered = true; + GetMaxSeqNo(); + } + + /// Work + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPartitionWriter::TEvWriteRequest, Handle); + hFunc(TEvPersQueue::TEvResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HoldPending(TEvPartitionWriter::TEvWriteRequest::TPtr& ev) { + auto& record = ev->Get()->Record; + const auto cookie = record.GetPartitionRequest().GetCookie(); + + Y_VERIFY(Pending.empty() || Pending.rbegin()->first < cookie); + Y_VERIFY(PendingReserve.empty() || PendingReserve.rbegin()->first < cookie); + Y_VERIFY(PendingWrite.empty() || PendingWrite.back() < cookie); + + Pending.emplace(cookie, std::move(ev->Get()->Record)); + } + + void Handle(TEvPartitionWriter::TEvWriteRequest::TPtr& ev) { + HoldPending(ev); + ReserveBytes(); + } + + void ReserveBytes() { + while (Pending) { + auto it = Pending.begin(); + + FillHeader(*it->second.MutablePartitionRequest(), PartitionId, PipeClient, OwnerCookie, it->first); + auto ev = MakeRequest(PartitionId, PipeClient, OwnerCookie, it->first); + + auto& request = *ev->Record.MutablePartitionRequest(); + request.SetMessageNo(MessageNo++); + + auto& cmd = *request.MutableCmdReserveBytes(); + cmd.SetSize(it->second.ByteSize()); + cmd.SetLastRequest(false); + + NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); + + PendingReserve.emplace(it->first, std::move(it->second)); + Pending.erase(it); + } + } + + void Write(ui64 cookie) { + Y_VERIFY(!PendingReserve.empty()); + auto it = PendingReserve.begin(); + + Y_VERIFY(it->first == cookie); + Y_VERIFY(PendingWrite.empty() || PendingWrite.back() < cookie); + + auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); + ev->Record = std::move(it->second); + + auto& request = *ev->Record.MutablePartitionRequest(); + request.SetMessageNo(MessageNo++); + + NTabletPipe::SendData(SelfId(), PipeClient, ev.Release()); + + PendingWrite.emplace_back(cookie); + PendingReserve.erase(it); + } + + void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + + TString error; + if (!BasicCheck(record, error)) { + return WriteResult(error, std::move(record)); + } + + const auto& response = record.GetPartitionResponse(); + if (!response.CmdWriteResultSize()) { + if (PendingReserve.empty()) { + return WriteResult("Unexpected ReserveBytes response", std::move(record)); + } + + const auto cookie = PendingReserve.begin()->first; + if (cookie != response.GetCookie()) { + error = TStringBuilder() << "Unexpected cookie at ReserveBytes" + << ": expected# " << cookie + << ", got# " << response.GetCookie(); + return WriteResult(error, std::move(record)); + } + + WriteAccepted(cookie); + Write(cookie); + } else { + if (PendingWrite.empty()) { + return WriteResult("Unexpected Write response", std::move(record)); + } + + const auto cookie = PendingWrite.front(); + if (cookie != response.GetCookie()) { + error = TStringBuilder() << "Unexpected cookie at Write" + << ": expected# " << cookie + << ", got# " << response.GetCookie(); + return WriteResult(error, std::move(record)); + } + + WriteResult(std::move(record)); + } + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->TabletId == TabletId && ev->Get()->Status != NKikimrProto::OK) { + Disconnected(); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { + if (ev->Get()->TabletId == TabletId) { + Disconnected(); + } + } + + void PassAway() override { + if (PipeClient) { + NTabletPipe::CloseAndForgetClient(SelfId(), PipeClient); + } + + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::PQ_PARTITION_WRITER_ACTOR; + } + + explicit TPartitionWriter( + const TActorId& client, + ui64 tabletId, + ui32 partitionId, + const TString& sourceId, + const TPartitionWriterOpts& opts) + : Client(client) + , TabletId(tabletId) + , PartitionId(partitionId) + , SourceId(sourceId) + , Opts(opts) + { + } + + void Bootstrap() { + NTabletPipe::TClientConfig config; + config.RetryPolicy = { + .RetryLimitCount = 6, + .MinRetryTime = TDuration::MilliSeconds(10), + .MaxRetryTime = TDuration::MilliSeconds(100), + .BackoffMultiplier = 2, + .DoFirstRetryInstantly = true + }; + + PipeClient = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), TabletId, config)); + + GetOwnership(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + + STATEFN(StateZombie) { + switch (ev->GetTypeRewrite()) { + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Client; + const ui64 TabletId; + const ui32 PartitionId; + const TString SourceId; + const TPartitionWriterOpts Opts; + + TActorId PipeClient; + TString OwnerCookie; + bool Registered = false; + ui64 MessageNo = 0; + + TMap<ui64, NKikimrClient::TPersQueueRequest> Pending; + TMap<ui64, NKikimrClient::TPersQueueRequest> PendingReserve; + TDeque<ui64> PendingWrite; + +}; // TPartitionWriter + +IActor* CreatePartitionWriter(const TActorId& client, ui64 tabletId, ui32 partitionId, const TString& sourceId, const TPartitionWriterOpts& opts) { + return new TPartitionWriter(client, tabletId, partitionId, sourceId, opts); +} + +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/writer/writer.h b/ydb/core/persqueue/writer/writer.h index fe339749fa7..5e880bab00b 100644 --- a/ydb/core/persqueue/writer/writer.h +++ b/ydb/core/persqueue/writer/writer.h @@ -1,122 +1,122 @@ -#pragma once - +#pragma once + #include <ydb/core/base/defs.h> #include <ydb/core/base/events.h> #include <ydb/core/protos/msgbus.pb.h> #include <ydb/core/protos/msgbus_pq.pb.h> - -#include <variant> - -namespace NKikimr { -namespace NPQ { - -struct TEvPartitionWriter { - enum EEv { - EvInitResult = EventSpaceBegin(TKikimrEvents::ES_PQ_PARTITION_WRITER), - EvWriteRequest, - EvWriteAccepted, - EvWriteResponse, - EvDisconnected, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_PARTITION_WRITER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_PARTITION_WRITER)"); - - struct TEvInitResult: public TEventLocal<TEvInitResult, EvInitResult> { - using TSourceIdInfo = NKikimrClient::TPersQueuePartitionResponse::TCmdGetMaxSeqNoResult::TSourceIdInfo; - - struct TSuccess { - TString OwnerCookie; - TSourceIdInfo SourceIdInfo; - }; - - struct TError { - TString Reason; - NKikimrClient::TResponse Response; - }; - - std::variant<TSuccess, TError> Result; - - explicit TEvInitResult(const TString& ownerCookie, const TSourceIdInfo& sourceIdInfo) - : Result(TSuccess{ownerCookie, sourceIdInfo}) - { - } - - explicit TEvInitResult(const TString& reason, NKikimrClient::TResponse&& response) - : Result(TError{reason, std::move(response)}) - { - } - - bool IsSuccess() const { return Result.index() == 0; } - const TSuccess& GetResult() const { return std::get<0>(Result); } - const TError& GetError() const { return std::get<1>(Result); } - TString ToString() const override; - }; - - struct TEvWriteRequest: public TEventPB<TEvWriteRequest, NKikimrClient::TPersQueueRequest, EvWriteRequest> { - // Only Cookie & CmdWrite must be set, other fields can be overwritten - TEvWriteRequest() = default; - - explicit TEvWriteRequest(ui64 cookie) { - Record.MutablePartitionRequest()->SetCookie(cookie); - } - }; - - struct TEvWriteAccepted: public TEventLocal<TEvWriteAccepted, EvWriteAccepted> { - ui64 Cookie; - - explicit TEvWriteAccepted(ui64 cookie) - : Cookie(cookie) - { - } - - TString ToString() const override; - }; - - struct TEvWriteResponse: public TEventPB<TEvWriteResponse, NKikimrClient::TResponse, EvWriteResponse> { - struct TSuccess { - }; - - struct TError { - TString Reason; - }; - - std::variant<TSuccess, TError> Result; - - TEvWriteResponse() = default; - - explicit TEvWriteResponse(NKikimrClient::TResponse&& response) - : Result(TSuccess{}) - { - Record = std::move(response); - } - - explicit TEvWriteResponse(const TString& reason, NKikimrClient::TResponse&& response) - : Result(TError{reason}) - { - Record = std::move(response); - } - - bool IsSuccess() const { return Result.index() == 0; } - const TError& GetError() const { return std::get<1>(Result); } - }; - - struct TEvDisconnected: public TEventLocal<TEvDisconnected, EvDisconnected> { - }; - -}; // TEvPartitionWriter - -struct TPartitionWriterOpts { - bool CheckState = false; - bool AutoRegister = false; - - TPartitionWriterOpts& WithCheckState(bool value) { CheckState = value; return *this; } - TPartitionWriterOpts& WithAutoRegister(bool value) { AutoRegister = value; return *this; } -}; - -IActor* CreatePartitionWriter(const TActorId& client, ui64 tabletId, ui32 partitionId, const TString& sourceId, - const TPartitionWriterOpts& opts = {}); - -} // NPQ -} // NKikimr + +#include <variant> + +namespace NKikimr { +namespace NPQ { + +struct TEvPartitionWriter { + enum EEv { + EvInitResult = EventSpaceBegin(TKikimrEvents::ES_PQ_PARTITION_WRITER), + EvWriteRequest, + EvWriteAccepted, + EvWriteResponse, + EvDisconnected, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_PARTITION_WRITER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_PARTITION_WRITER)"); + + struct TEvInitResult: public TEventLocal<TEvInitResult, EvInitResult> { + using TSourceIdInfo = NKikimrClient::TPersQueuePartitionResponse::TCmdGetMaxSeqNoResult::TSourceIdInfo; + + struct TSuccess { + TString OwnerCookie; + TSourceIdInfo SourceIdInfo; + }; + + struct TError { + TString Reason; + NKikimrClient::TResponse Response; + }; + + std::variant<TSuccess, TError> Result; + + explicit TEvInitResult(const TString& ownerCookie, const TSourceIdInfo& sourceIdInfo) + : Result(TSuccess{ownerCookie, sourceIdInfo}) + { + } + + explicit TEvInitResult(const TString& reason, NKikimrClient::TResponse&& response) + : Result(TError{reason, std::move(response)}) + { + } + + bool IsSuccess() const { return Result.index() == 0; } + const TSuccess& GetResult() const { return std::get<0>(Result); } + const TError& GetError() const { return std::get<1>(Result); } + TString ToString() const override; + }; + + struct TEvWriteRequest: public TEventPB<TEvWriteRequest, NKikimrClient::TPersQueueRequest, EvWriteRequest> { + // Only Cookie & CmdWrite must be set, other fields can be overwritten + TEvWriteRequest() = default; + + explicit TEvWriteRequest(ui64 cookie) { + Record.MutablePartitionRequest()->SetCookie(cookie); + } + }; + + struct TEvWriteAccepted: public TEventLocal<TEvWriteAccepted, EvWriteAccepted> { + ui64 Cookie; + + explicit TEvWriteAccepted(ui64 cookie) + : Cookie(cookie) + { + } + + TString ToString() const override; + }; + + struct TEvWriteResponse: public TEventPB<TEvWriteResponse, NKikimrClient::TResponse, EvWriteResponse> { + struct TSuccess { + }; + + struct TError { + TString Reason; + }; + + std::variant<TSuccess, TError> Result; + + TEvWriteResponse() = default; + + explicit TEvWriteResponse(NKikimrClient::TResponse&& response) + : Result(TSuccess{}) + { + Record = std::move(response); + } + + explicit TEvWriteResponse(const TString& reason, NKikimrClient::TResponse&& response) + : Result(TError{reason}) + { + Record = std::move(response); + } + + bool IsSuccess() const { return Result.index() == 0; } + const TError& GetError() const { return std::get<1>(Result); } + }; + + struct TEvDisconnected: public TEventLocal<TEvDisconnected, EvDisconnected> { + }; + +}; // TEvPartitionWriter + +struct TPartitionWriterOpts { + bool CheckState = false; + bool AutoRegister = false; + + TPartitionWriterOpts& WithCheckState(bool value) { CheckState = value; return *this; } + TPartitionWriterOpts& WithAutoRegister(bool value) { AutoRegister = value; return *this; } +}; + +IActor* CreatePartitionWriter(const TActorId& client, ui64 tabletId, ui32 partitionId, const TString& sourceId, + const TPartitionWriterOpts& opts = {}); + +} // NPQ +} // NKikimr diff --git a/ydb/core/persqueue/writer/ya.make b/ydb/core/persqueue/writer/ya.make index fcf247732eb..6cc1bb572ed 100644 --- a/ydb/core/persqueue/writer/ya.make +++ b/ydb/core/persqueue/writer/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - ilnaz +LIBRARY() + +OWNER( + ilnaz g:kikimr - g:logbroker -) - -SRCS( - source_id_encoding.cpp - writer.cpp -) - -PEERDIR( - library/cpp/actors/core - library/cpp/string_utils/base64 + g:logbroker +) + +SRCS( + source_id_encoding.cpp + writer.cpp +) + +PEERDIR( + library/cpp/actors/core + library/cpp/string_utils/base64 ydb/core/base ydb/core/persqueue/events ydb/core/protos ydb/public/lib/base -) - -END() +) + +END() diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 240e10e2492..50e89676ef9 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -30,8 +30,8 @@ SRCS( actor_persqueue_client_iface.h ) -GENERATE_ENUM_SERIALIZATION(sourceid.h) - +GENERATE_ENUM_SERIALIZATION(sourceid.h) + PEERDIR( library/cpp/actors/core library/cpp/html/pcdata diff --git a/ydb/core/protos/base.proto b/ydb/core/protos/base.proto index f045e35bfeb..1f7e855e091 100644 --- a/ydb/core/protos/base.proto +++ b/ydb/core/protos/base.proto @@ -37,8 +37,8 @@ message TRowVersion { optional uint64 Step = 1; optional uint64 TxId = 2; } - -message TPathID { - optional fixed64 OwnerId = 1; - optional uint64 LocalId = 2; -} + +message TPathID { + optional fixed64 OwnerId = 1; + optional uint64 LocalId = 2; +} diff --git a/ydb/core/protos/change_exchange.proto b/ydb/core/protos/change_exchange.proto index 8579d3e2986..732c4716c2c 100644 --- a/ydb/core/protos/change_exchange.proto +++ b/ydb/core/protos/change_exchange.proto @@ -1,102 +1,102 @@ -import "google/protobuf/empty.proto"; - -package NKikimrChangeExchange; -option java_package = "ru.yandex.kikimr.proto"; - -message TChangeRecord { - message TDataChange { - message TSerializedCells { - // Column tags - repeated uint32 Tags = 1; - // Serialized values (TSerializedCellVec) - optional bytes Data = 2; - } - - /* key columns */ - optional TSerializedCells Key = 1; - - /* value columns (doesn't include key columns) */ - // old image - optional TSerializedCells OldImage = 2; - // new image - optional TSerializedCells NewImage = 3; - // Row operation - // Cells (if any) can be empty if: - // - there is no change in data-columns of AsyncIndex; - // - there is no need to save update to Cdc Stream (e.g. in "keys only" mode). - oneof RowOperation { - // apply specified values - TSerializedCells Upsert = 4; - // remove entire row - google.protobuf.Empty Erase = 5; - // reset all columns & apply specified values - TSerializedCells Reset = 6; - } - } - - optional uint64 Order = 1; - optional uint64 Group = 2; - optional uint64 Step = 3; - optional uint64 TxId = 4; - optional uint64 PathOwnerId = 5; - optional uint64 LocalPathId = 6; - oneof Kind { - TDataChange AsyncIndex = 7; - TDataChange CdcDataChange = 8; - // TSchemaChange SchemaChange = 9; - } -} - -message TEvHandshake { - // tablet id - optional uint64 Origin = 1; - // tablet generation - optional uint64 Generation = 2; -} - -message TEvApplyRecords { - // tablet id - optional uint64 Origin = 1; - // tablet generation - optional uint64 Generation = 2; - // records - repeated TChangeRecord Records = 3; -} - -message TEvStatus { - enum EStatus { - STATUS_UNKNOWN = 0; - STATUS_OK = 1; - STATUS_REJECT = 2; - } - - enum EReason { - REASON_NONE = 0; - REASON_UNKNOWN_ORIGIN = 1; - REASON_STALE_ORIGIN = 2; - REASON_ORDER_VIOLATION = 3; - REASON_UNEXPECTED_KIND = 4; - REASON_UNEXPECTED_ROW_OPERATION = 5; - REASON_SCHEME_ERROR = 6; - REASON_WRONG_STATE = 7; - } - - message TRecordStatus { - optional uint64 Order = 1; - optional EStatus Status = 2; - optional EReason Reason = 3; - } - - optional EStatus Status = 1; - optional EReason Reason = 2; - repeated TRecordStatus RecordStatuses = 3; - optional uint64 LastRecordOrder = 4; -} - -message TEvActivateSender { - optional uint64 Origin = 1; -} - -message TEvActivateSenderAck { - optional uint64 Origin = 1; -} +import "google/protobuf/empty.proto"; + +package NKikimrChangeExchange; +option java_package = "ru.yandex.kikimr.proto"; + +message TChangeRecord { + message TDataChange { + message TSerializedCells { + // Column tags + repeated uint32 Tags = 1; + // Serialized values (TSerializedCellVec) + optional bytes Data = 2; + } + + /* key columns */ + optional TSerializedCells Key = 1; + + /* value columns (doesn't include key columns) */ + // old image + optional TSerializedCells OldImage = 2; + // new image + optional TSerializedCells NewImage = 3; + // Row operation + // Cells (if any) can be empty if: + // - there is no change in data-columns of AsyncIndex; + // - there is no need to save update to Cdc Stream (e.g. in "keys only" mode). + oneof RowOperation { + // apply specified values + TSerializedCells Upsert = 4; + // remove entire row + google.protobuf.Empty Erase = 5; + // reset all columns & apply specified values + TSerializedCells Reset = 6; + } + } + + optional uint64 Order = 1; + optional uint64 Group = 2; + optional uint64 Step = 3; + optional uint64 TxId = 4; + optional uint64 PathOwnerId = 5; + optional uint64 LocalPathId = 6; + oneof Kind { + TDataChange AsyncIndex = 7; + TDataChange CdcDataChange = 8; + // TSchemaChange SchemaChange = 9; + } +} + +message TEvHandshake { + // tablet id + optional uint64 Origin = 1; + // tablet generation + optional uint64 Generation = 2; +} + +message TEvApplyRecords { + // tablet id + optional uint64 Origin = 1; + // tablet generation + optional uint64 Generation = 2; + // records + repeated TChangeRecord Records = 3; +} + +message TEvStatus { + enum EStatus { + STATUS_UNKNOWN = 0; + STATUS_OK = 1; + STATUS_REJECT = 2; + } + + enum EReason { + REASON_NONE = 0; + REASON_UNKNOWN_ORIGIN = 1; + REASON_STALE_ORIGIN = 2; + REASON_ORDER_VIOLATION = 3; + REASON_UNEXPECTED_KIND = 4; + REASON_UNEXPECTED_ROW_OPERATION = 5; + REASON_SCHEME_ERROR = 6; + REASON_WRONG_STATE = 7; + } + + message TRecordStatus { + optional uint64 Order = 1; + optional EStatus Status = 2; + optional EReason Reason = 3; + } + + optional EStatus Status = 1; + optional EReason Reason = 2; + repeated TRecordStatus RecordStatuses = 3; + optional uint64 LastRecordOrder = 4; +} + +message TEvActivateSender { + optional uint64 Origin = 1; +} + +message TEvActivateSenderAck { + optional uint64 Origin = 1; +} diff --git a/ydb/core/protos/cms.proto b/ydb/core/protos/cms.proto index b506fadb65f..fa3e2452191 100644 --- a/ydb/core/protos/cms.proto +++ b/ydb/core/protos/cms.proto @@ -18,7 +18,7 @@ message TStatus { UNAUTHORIZED = 9; NO_SUCH_HOST = 10; NO_SUCH_DEVICE = 11; - NO_SUCH_SERVICE = 12; + NO_SUCH_SERVICE = 12; } optional ECode Code = 1; @@ -111,8 +111,8 @@ message TAction { repeated string Services = 3; repeated string Devices = 4; optional uint64 Duration = 5; - // If specified will be expanded to list of hosts. - optional string Tenant = 6; + // If specified will be expanded to list of hosts. + optional string Tenant = 6; } enum ETenantPolicy { @@ -398,33 +398,33 @@ message TCmsConfig { optional uint64 FaultyPrepTimeout = 6 [default = 3600000000]; } - message TSentinelConfig { - message TStateLimit { - optional uint32 State = 1; // EPDiskState - optional uint32 Limit = 2; - } - - optional bool Enable = 1; - // Updater's config - optional uint64 UpdateConfigInterval = 2 [default = 3600000000]; - optional uint64 RetryUpdateConfig = 3 [default = 60000000]; - optional uint64 UpdateStateInterval = 4 [default = 60000000]; - optional uint64 UpdateStateTimeout = 5 [default = 45000000]; - // Status changer's config - optional uint64 RetryChangeStatus = 6 [default = 10000000]; - optional uint32 ChangeStatusRetries = 7 [default = 5]; - // Status computer's config - optional uint32 DefaultStateLimit = 8 [default = 5]; - optional uint32 BrokenLimit = 9 [default = 60]; // unused - // Guardian's config - optional uint32 DataCenterRatio = 10 [default = 50]; - optional uint32 RoomRatio = 11 [default = 70]; - optional uint32 RackRatio = 12 [default = 90]; - - optional bool DryRun = 13; - repeated TStateLimit StateLimits = 14; - } - + message TSentinelConfig { + message TStateLimit { + optional uint32 State = 1; // EPDiskState + optional uint32 Limit = 2; + } + + optional bool Enable = 1; + // Updater's config + optional uint64 UpdateConfigInterval = 2 [default = 3600000000]; + optional uint64 RetryUpdateConfig = 3 [default = 60000000]; + optional uint64 UpdateStateInterval = 4 [default = 60000000]; + optional uint64 UpdateStateTimeout = 5 [default = 45000000]; + // Status changer's config + optional uint64 RetryChangeStatus = 6 [default = 10000000]; + optional uint32 ChangeStatusRetries = 7 [default = 5]; + // Status computer's config + optional uint32 DefaultStateLimit = 8 [default = 5]; + optional uint32 BrokenLimit = 9 [default = 60]; // unused + // Guardian's config + optional uint32 DataCenterRatio = 10 [default = 50]; + optional uint32 RoomRatio = 11 [default = 70]; + optional uint32 RackRatio = 12 [default = 90]; + + optional bool DryRun = 13; + repeated TStateLimit StateLimits = 14; + } + message TLogConfig { enum ELevel { ENABLED = 0; @@ -441,14 +441,14 @@ message TCmsConfig { optional uint64 TTL = 3 [default = 1209600000000]; // 14 days } - optional uint64 DefaultRetryTime = 1 [default = 300000000]; - optional uint64 DefaultPermissionDuration = 2 [default = 300000000]; - optional TMonitorConfig MonitorConfig = 3; - optional TLimits TenantLimits = 4; - optional TLimits ClusterLimits = 5; - optional uint64 InfoCollectionTimeout = 6 [default = 15000000]; - optional TLogConfig LogConfig = 7; - optional TSentinelConfig SentinelConfig = 8; + optional uint64 DefaultRetryTime = 1 [default = 300000000]; + optional uint64 DefaultPermissionDuration = 2 [default = 300000000]; + optional TMonitorConfig MonitorConfig = 3; + optional TLimits TenantLimits = 4; + optional TLimits ClusterLimits = 5; + optional uint64 InfoCollectionTimeout = 6 [default = 15000000]; + optional TLogConfig LogConfig = 7; + optional TSentinelConfig SentinelConfig = 8; } message TPDiskID { @@ -532,8 +532,8 @@ message TLogRecordData { repeated EMarker NodeMarkers = 5; optional TAvailabilityStats DiskAvailabilityStats = 6; optional TAvailabilityStats NodeAvailabilityStats = 7; - optional string Host = 8; - optional string Reason = 9; + optional string Host = 8; + optional string Reason = 9; } message TMarkersModification { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 1b359fc2afa..d64169d4fc0 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -150,7 +150,7 @@ message TStaticNameserviceConfig { message TDynamicNameserviceConfig { optional uint32 MaxStaticNodeId = 1 [default = 1000]; - optional uint32 MaxDynamicNodeId = 2 [default = 200000]; + optional uint32 MaxDynamicNodeId = 2 [default = 200000]; optional uint64 LeaseDuration = 3 [default = 3600000000]; // DEPRECATED } @@ -621,16 +621,16 @@ message TFeatureFlags { optional bool TrimEntireDeviceOnStartup = 6 [default = false]; optional bool EnableChunkGraceForPDisk = 7 [default = true]; optional bool AllowConsistentOperationsForSchemeShard = 8 [default = true]; - optional bool EnableSchemeBoard = 9 [default = true]; // deprecated: always true + optional bool EnableSchemeBoard = 9 [default = true]; // deprecated: always true optional bool AllowYdbRequestsWithoutDatabase = 10 [default = true]; optional bool EnableExternalSubdomains = 11 [default = true]; - optional bool AllowRecursiveMkDir = 12 [default = true]; // deprecated: always true + optional bool AllowRecursiveMkDir = 12 [default = true]; // deprecated: always true optional bool AllowHugeKeyValueDeletes = 13 [default = true]; // delete when all clients limit deletes per request optional bool SendSchemaVersionToDatashard = 14 [default = true]; // deprecated: always true - optional bool EnableSchemeBoardCache = 15 [default = true]; // deprecated: always true + optional bool EnableSchemeBoardCache = 15 [default = true]; // deprecated: always true optional bool EnableSystemViews = 16 [default = true]; optional bool EnableExternalHive = 17 [default = true]; - optional bool UseSchemeBoardCacheForSchemeRequests = 18 [default = true]; // deprecated: always true + optional bool UseSchemeBoardCacheForSchemeRequests = 18 [default = true]; // deprecated: always true optional bool CompileMinikqlWithVersion = 19 [default = true]; // deprecated: always true optional bool ReadTableWithSnapshot = 20 [default = true]; optional bool ImportantTabletsUseSystemPool = 21 [default = true]; @@ -642,9 +642,9 @@ message TFeatureFlags { optional bool EnablePutBatchingForBlobStorage = 27 [default = true]; optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true optional bool EnableKqpScanQueries = 29 [default = true]; // deprecated: always true - optional bool EnablePersistentPartitionStats = 30 [default = false]; - optional bool EnableTtlOnIndexedTables = 31 [default = true]; // deprecated: always true - optional bool AllowUpdateChannelsBindingOfSolomonPartitions = 32 [default = false]; + optional bool EnablePersistentPartitionStats = 30 [default = false]; + optional bool EnableTtlOnIndexedTables = 31 [default = true]; // deprecated: always true + optional bool AllowUpdateChannelsBindingOfSolomonPartitions = 32 [default = false]; optional bool DisableLegacyYql = 33 [default = true]; // deprecated: always true optional bool EnableDataColumnForIndexTable = 34 [default = true]; optional bool AllowServerlessStorageBillingForSchemeShard = 35 [default = false]; @@ -653,7 +653,7 @@ message TFeatureFlags { optional bool EnableConfigurationCache = 38 [default = false]; optional bool EnableDbCounters = 39 [default = false]; optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false]; - optional bool EnableAsyncIndexes = 41 [default = true]; + optional bool EnableAsyncIndexes = 41 [default = true]; optional bool AllowStreamExecuteYqlScript = 42 [default = true]; optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true optional bool EnableOlapSchemaOperations = 44 [default = false]; @@ -667,8 +667,8 @@ message TFeatureFlags { optional bool EnableArrowFormatInChannels = 52 [default = false]; optional bool EnableBackgroundCompactionServerless = 53 [default = false]; optional bool EnableNotNullColumns = 54 [default = false]; - optional bool EnableTtlOnAsyncIndexedTables = 55 [default = false]; - optional bool EnableBulkUpsertToAsyncIndexedTables = 56 [default = false]; + optional bool EnableTtlOnAsyncIndexedTables = 55 [default = false]; + optional bool EnableBulkUpsertToAsyncIndexedTables = 56 [default = false]; optional bool EnableNodeBrokerSingleDomainMode = 57 [default = false]; optional bool EnableKqpSessionActor = 58 [default = false]; // Dangerous settings we don't want in the public api unless required @@ -1189,7 +1189,7 @@ message TImmediateControlsConfig { message TMeteringConfig { optional string MeteringFilePath = 1; - repeated string SystemBackupSIDs = 2; + repeated string SystemBackupSIDs = 2; }; message THiveTabletLimit { @@ -1290,10 +1290,10 @@ message TDataShardConfig { optional uint64 BackupReadAheadLo = 3 [default = 524288]; optional uint64 BackupReadAheadHi = 4 [default = 1048576]; optional uint64 KeepSnapshotTimeout = 5 [default = 300000]; // milliseconds - optional uint64 ChangesQueueItemsLimit = 6 [default = 10000]; - optional uint64 ChangesQueueBytesLimit = 7 [default = 131072000]; // 125 MB - optional string RestoreTaskName = 8 [default = "restore"]; - optional uint32 RestoreTaskPriority = 9 [default = 10]; + optional uint64 ChangesQueueItemsLimit = 6 [default = 10000]; + optional uint64 ChangesQueueBytesLimit = 7 [default = 131072000]; // 125 MB + optional string RestoreTaskName = 8 [default = "restore"]; + optional uint32 RestoreTaskPriority = 9 [default = 10]; optional uint64 CleanupSnapshotPeriod = 10 [default = 15000]; // milliseconds } diff --git a/ydb/core/protos/counters_datashard.proto b/ydb/core/protos/counters_datashard.proto index 26b5c317482..956a1369378 100644 --- a/ydb/core/protos/counters_datashard.proto +++ b/ydb/core/protos/counters_datashard.proto @@ -93,10 +93,10 @@ enum ECumulativeCounters { COUNTER_BULK_UPSERT_SUCCESS = 67 [(CounterOpts) = {Name: "BulkUpsertSuccess"}]; COUNTER_BULK_UPSERT_ERROR = 68 [(CounterOpts) = {Name: "BulkUpsertError"}]; COUNTER_BULK_UPSERT_OVERLOADED = 69 [(CounterOpts) = {Name: "BulkUpsertOverloaded"}]; - COUNTER_ERASE_ROWS_SUCCESS = 70 [(CounterOpts) = {Name: "EraseRowsSuccess"}]; - COUNTER_ERASE_ROWS_ERROR = 71 [(CounterOpts) = {Name: "EraseRowsError"}]; - COUNTER_ERASE_ROWS_OVERLOADED = 72 [(CounterOpts) = {Name: "EraseRowsOverloaded"}]; - COUNTER_ERASE_ROWS = 73 [(CounterOpts) = {Name: "EraseRows"}]; + COUNTER_ERASE_ROWS_SUCCESS = 70 [(CounterOpts) = {Name: "EraseRowsSuccess"}]; + COUNTER_ERASE_ROWS_ERROR = 71 [(CounterOpts) = {Name: "EraseRowsError"}]; + COUNTER_ERASE_ROWS_OVERLOADED = 72 [(CounterOpts) = {Name: "EraseRowsOverloaded"}]; + COUNTER_ERASE_ROWS = 73 [(CounterOpts) = {Name: "EraseRows"}]; COUNTER_BUILD_DEPENDENCIES_USEC = 74 [(CounterOpts) = {Name: "BuildDependenciesMicrosec"}]; COUNTER_APPLY_LOCKS_USEC = 75 [(CounterOpts) = {Name: "ApplyLocksMicrosec"}]; COUNTER_TX_NOT_ENOUGH_MEMORY_1 = 76 [(CounterOpts) = {Name: "TxNotEnoughMemory1"}]; @@ -392,28 +392,28 @@ enum ETxTypes { TXTYPE_DISCARD_VOLATILE_SNAPSHOT = 43 [(TxTypeOpts) = {Name: "TxDiscardVolatileSnapshot"}]; TXTYPE_CLEANUP_REMOVED_SNAPSHOTS = 44 [(TxTypeOpts) = {Name: "TxCleanupRemovedSnapshots"}]; TXTYPE_MIGRATE_SCHEMESHARD = 45 [(TxTypeOpts) = {Name: "TxMigrateSchemeShard"}]; - TXTYPE_GET_S3_UPLOAD_ID = 46 [(TxTypeOpts) = {Name: "TxGetS3UploadId"}]; - TXTYPE_STORE_S3_UPLOAD_ID = 47 [(TxTypeOpts) = {Name: "TxStoreS3UploadId"}]; - TXTYPE_ERASE_ROWS = 48 [(TxTypeOpts) = {Name: "TxEraseRows"}]; + TXTYPE_GET_S3_UPLOAD_ID = 46 [(TxTypeOpts) = {Name: "TxGetS3UploadId"}]; + TXTYPE_STORE_S3_UPLOAD_ID = 47 [(TxTypeOpts) = {Name: "TxStoreS3UploadId"}]; + TXTYPE_ERASE_ROWS = 48 [(TxTypeOpts) = {Name: "TxEraseRows"}]; TXTYPE_RELEASE_SNAPSHOT_REFERENCE = 49 [(TxTypeOpts) = {Name: "TxReleaseSnaphotReference"}]; TXTYPE_STOP_GUARD = 50 [(TxTypeOpts) = {Name: "TxStopGuard"}]; - TXTYPE_GET_S3_DOWNLOAD_INFO = 51 [(TxTypeOpts) = {Name: "TxGetS3DownloadInfo"}]; - TXTYPE_STORE_S3_DOWNLOAD_INFO = 52 [(TxTypeOpts) = {Name: "TxStoreS3DownloadInfo"}]; - TXTYPE_UNSAFE_UPLOAD_ROWS = 53 [(TxTypeOpts) = {Name: "TxUnsafeUploadRows"}]; + TXTYPE_GET_S3_DOWNLOAD_INFO = 51 [(TxTypeOpts) = {Name: "TxGetS3DownloadInfo"}]; + TXTYPE_STORE_S3_DOWNLOAD_INFO = 52 [(TxTypeOpts) = {Name: "TxStoreS3DownloadInfo"}]; + TXTYPE_UNSAFE_UPLOAD_ROWS = 53 [(TxTypeOpts) = {Name: "TxUnsafeUploadRows"}]; TXTYPE_PERSIST_SUBDOMAIN_PATH_ID = 54 [(TxTypeOpts) = {Name: "TxPersistSubDomainPathId"}]; TXTYPE_PERSIST_SUBDOMAIN_OUT_OF_SPACE = 55 [(TxTypeOpts) = {Name: "TxPersistSubDomainOutOfSpace"}]; - TXTYPE_REQUEST_CHANGE_RECORDS = 56 [(TxTypeOpts) = {Name: "TxRequestChangeRecords"}]; - TXTYPE_REMOVE_CHANGE_RECORDS = 57 [(TxTypeOpts) = {Name: "TxRemoveChangeRecords"}]; - TXTYPE_CHANGE_EXCHANGE_HANDSHAKE = 58 [(TxTypeOpts) = {Name: "TxChangeExchangeHandshake"}]; - TXTYPE_APPLY_CHANGE_RECORDS = 59 [(TxTypeOpts) = {Name: "TxApplyChangeRecords"}]; + TXTYPE_REQUEST_CHANGE_RECORDS = 56 [(TxTypeOpts) = {Name: "TxRequestChangeRecords"}]; + TXTYPE_REMOVE_CHANGE_RECORDS = 57 [(TxTypeOpts) = {Name: "TxRemoveChangeRecords"}]; + TXTYPE_CHANGE_EXCHANGE_HANDSHAKE = 58 [(TxTypeOpts) = {Name: "TxChangeExchangeHandshake"}]; + TXTYPE_APPLY_CHANGE_RECORDS = 59 [(TxTypeOpts) = {Name: "TxApplyChangeRecords"}]; TXTYPE_EXECUTE_MVCC_STATE_CHANGE = 60 [(TxTypeOpts) = {Name: "TxExecuteMvccStateChange"}]; - TXTYPE_CHANGE_S3_UPLOAD_STATUS = 61 [(TxTypeOpts) = {Name: "TxChangeS3UploadStatus"}]; - TXTYPE_ACTIVATE_CHANGE_SENDER = 62 [(TxTypeOpts) = {Name: "TxActivateChangeSender"}]; - TXTYPE_ACTIVATE_CHANGE_SENDER_ACK = 63 [(TxTypeOpts) = {Name: "TxActivateChangeSenderAck"}]; + TXTYPE_CHANGE_S3_UPLOAD_STATUS = 61 [(TxTypeOpts) = {Name: "TxChangeS3UploadStatus"}]; + TXTYPE_ACTIVATE_CHANGE_SENDER = 62 [(TxTypeOpts) = {Name: "TxActivateChangeSender"}]; + TXTYPE_ACTIVATE_CHANGE_SENDER_ACK = 63 [(TxTypeOpts) = {Name: "TxActivateChangeSenderAck"}]; TXTYPE_CLEANUP = 64 [(TxTypeOpts) = {Name: "TxCleanup"}]; TXTYPE_COMPACT_BORROWED = 65 [(TxTypeOpts) = {Name: "TxCompactBorrowed"}]; TXTYPE_COMPACT_TABLE = 66 [(TxTypeOpts) = {Name: "TxCompactTable"}]; - TXTYPE_CHANGE_EXCHANGE_SPLIT_ACK = 67 [(TxTypeOpts) = {Name: "TxChangeExchangeSplitAck"}]; + TXTYPE_CHANGE_EXCHANGE_SPLIT_ACK = 67 [(TxTypeOpts) = {Name: "TxChangeExchangeSplitAck"}]; TXTYPE_APPLY_REPLICATION_CHANGES = 68 [(TxTypeOpts) = {Name: "TxApplyReplicationChanges"}]; TXTYPE_READ = 69 [(TxTypeOpts) = {Name: "TxRead"}]; TXTYPE_SPLIT_REPLICATION_SOURCE_OFFSETS = 70 [(TxTypeOpts) = {Name: "TxSplitReplicationSourceOffsets"}]; diff --git a/ydb/core/protos/counters_pq.proto b/ydb/core/protos/counters_pq.proto index ac6f2c41025..72337581351 100644 --- a/ydb/core/protos/counters_pq.proto +++ b/ydb/core/protos/counters_pq.proto @@ -134,9 +134,9 @@ enum EPercentileCounters { COUNTER_LATENCY_PQ_RECEIVE_QUEUE = 15 [(CounterOpts) = {Name: "LatencyReciveQueue"}]; COUNTER_LATENCY_PQ_RESERVE_BYTES = 16 [(CounterOpts) = {Name: "LatencyReserveBytes"}]; - COUNTER_LATENCY_PQ_REGISTER_MESSAGE_GROUP = 17 [(CounterOpts) = {Name: "LatencyRegisterMessageGroup"}]; - COUNTER_LATENCY_PQ_DEREGISTER_MESSAGE_GROUP = 18 [(CounterOpts) = {Name: "LatencyDeregisterMessageGroup"}]; - COUNTER_LATENCY_PQ_SPLIT_MESSAGE_GROUP = 19 [(CounterOpts) = {Name: "LatencySplitMessageGroup"}]; + COUNTER_LATENCY_PQ_REGISTER_MESSAGE_GROUP = 17 [(CounterOpts) = {Name: "LatencyRegisterMessageGroup"}]; + COUNTER_LATENCY_PQ_DEREGISTER_MESSAGE_GROUP = 18 [(CounterOpts) = {Name: "LatencyDeregisterMessageGroup"}]; + COUNTER_LATENCY_PQ_SPLIT_MESSAGE_GROUP = 19 [(CounterOpts) = {Name: "LatencySplitMessageGroup"}]; } diff --git a/ydb/core/protos/counters_replication.proto b/ydb/core/protos/counters_replication.proto index 37892b0fcbc..b3c86ad17e5 100644 --- a/ydb/core/protos/counters_replication.proto +++ b/ydb/core/protos/counters_replication.proto @@ -1,34 +1,34 @@ import "ydb/core/protos/counters.proto"; - -package NKikimr.NReplication; -option java_package = "ru.yandex.kikimr.proto"; - -option (TabletTypeName) = "ReplicationController"; // Used as prefix for all counters - -enum ESimpleCounters { - COUNTER_SIMPLE_IGNORE = 0; -} - -enum ECumulativeCounters { - COUNTER_CUMULATIVE_IGNORE = 0; -} - -enum EPercentileCounters { - option (GlobalCounterOpts) = { - Ranges { Value: 0 Name: "0 ms" } - Ranges { Value: 1 Name: "1 ms" } - }; - - COUNTER_PERCENTILE_IGNORE = 0; -} - -enum ETxTypes { - TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}]; - TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}]; - TXTYPE_CREATE_REPLICATION = 2 [(TxTypeOpts) = {Name: "TxCreateReplication"}]; - TXTYPE_DROP_REPLICATION = 3 [(TxTypeOpts) = {Name: "TxDropReplication"}]; - TXTYPE_DISCOVERY_RESULT = 4 [(TxTypeOpts) = {Name: "TxDiscoveryResult"}]; - TXTYPE_ASSIGN_STREAM_NAME = 5 [(TxTypeOpts) = {Name: "TxAssignStreamName"}]; - TXTYPE_CREATE_STREAM_RESULT = 6 [(TxTypeOpts) = {Name: "TxCreateStreamResult"}]; - TXTYPE_CREATE_DST_RESULT = 7 [(TxTypeOpts) = {Name: "TxCreateDstResult"}]; -} + +package NKikimr.NReplication; +option java_package = "ru.yandex.kikimr.proto"; + +option (TabletTypeName) = "ReplicationController"; // Used as prefix for all counters + +enum ESimpleCounters { + COUNTER_SIMPLE_IGNORE = 0; +} + +enum ECumulativeCounters { + COUNTER_CUMULATIVE_IGNORE = 0; +} + +enum EPercentileCounters { + option (GlobalCounterOpts) = { + Ranges { Value: 0 Name: "0 ms" } + Ranges { Value: 1 Name: "1 ms" } + }; + + COUNTER_PERCENTILE_IGNORE = 0; +} + +enum ETxTypes { + TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}]; + TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}]; + TXTYPE_CREATE_REPLICATION = 2 [(TxTypeOpts) = {Name: "TxCreateReplication"}]; + TXTYPE_DROP_REPLICATION = 3 [(TxTypeOpts) = {Name: "TxDropReplication"}]; + TXTYPE_DISCOVERY_RESULT = 4 [(TxTypeOpts) = {Name: "TxDiscoveryResult"}]; + TXTYPE_ASSIGN_STREAM_NAME = 5 [(TxTypeOpts) = {Name: "TxAssignStreamName"}]; + TXTYPE_CREATE_STREAM_RESULT = 6 [(TxTypeOpts) = {Name: "TxCreateStreamResult"}]; + TXTYPE_CREATE_DST_RESULT = 7 [(TxTypeOpts) = {Name: "TxCreateDstResult"}]; +} diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index fde2a12b30f..a839915c98f 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -92,10 +92,10 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxCreateFileStore = 77 [(CounterOpts) = {Name: "InFlightOps/CreateFileStore"}]; COUNTER_IN_FLIGHT_OPS_TxAlterFileStore = 78 [(CounterOpts) = {Name: "InFlightOps/AlterFileStore"}]; COUNTER_IN_FLIGHT_OPS_TxDropFileStore = 79 [(CounterOpts) = {Name: "InFlightOps/DropFileStore"}]; - - COUNTER_TTL_ENABLED_TABLE_COUNT = 80 [(CounterOpts) = {Name: "TTLEnabledTables"}]; - - COUNTER_IN_FLIGHT_OPS_TxRestore = 81 [(CounterOpts) = {Name: "InFlightOps/Restore"}]; + + COUNTER_TTL_ENABLED_TABLE_COUNT = 80 [(CounterOpts) = {Name: "TTLEnabledTables"}]; + + COUNTER_IN_FLIGHT_OPS_TxRestore = 81 [(CounterOpts) = {Name: "InFlightOps/Restore"}]; COUNTER_IN_FLIGHT_OPS_TxCreateOlapStore = 82 [(CounterOpts) = {Name: "InFlightOps/CreateOlapStore"}]; COUNTER_IN_FLIGHT_OPS_TxAlterOlapStore = 83 [(CounterOpts) = {Name: "InFlightOps/AlterOlapStore"}]; @@ -113,7 +113,7 @@ enum ESimpleCounters { COUNTER_DISK_SPACE_TABLES_INDEX_BYTES = 93 [(CounterOpts) = {Name: "DiskSpaceTablesIndexBytes"}]; COUNTER_DISK_SPACE_TABLES_TOTAL_BYTES = 94 [(CounterOpts) = {Name: "DiskSpaceTablesTotalBytes"}]; COUNTER_DISK_SPACE_QUOTA_EXCEEDED = 95 [(CounterOpts) = {Name: "DiskSpaceQuotaExceeded"}]; - + COUNTER_DISK_SPACE_HARD_QUOTA_BYTES = 96 [(CounterOpts) = {Name: "DiskSpaceHardQuotaBytes"}]; COUNTER_DISK_SPACE_SOFT_QUOTA_BYTES = 97 [(CounterOpts) = {Name: "DiskSpaceSoftQuotaBytes"}]; @@ -142,12 +142,12 @@ enum ESimpleCounters { COUNTER_STREAM_RESERVED_THROUGHPUT = 115 [(CounterOpts) = {Name: "StreamReservedThroughput"}]; COUNTER_STREAM_RESERVED_STORAGE = 116 [(CounterOpts) = {Name: "StreamReservedStorage"}]; COUNTER_STREAM_RESERVED_STORAGE_QUOTA = 117 [(CounterOpts) = {Name: "StreamReservedStorageQuota"}]; - - COUNTER_REPLICATION_COUNT = 118 [(CounterOpts) = {Name: "Replications"}]; - COUNTER_REPLICATION_CONTROLLER_COUNT = 119 [(CounterOpts) = {Name: "ReplicationControllers"}]; - COUNTER_IN_FLIGHT_OPS_TxCreateReplication = 120 [(CounterOpts) = {Name: "InFlightOps/CreateReplication"}]; - COUNTER_IN_FLIGHT_OPS_TxAlterReplication = 121 [(CounterOpts) = {Name: "InFlightOps/AlterReplication"}]; - COUNTER_IN_FLIGHT_OPS_TxDropReplication = 122 [(CounterOpts) = {Name: "InFlightOps/DropReplication"}]; + + COUNTER_REPLICATION_COUNT = 118 [(CounterOpts) = {Name: "Replications"}]; + COUNTER_REPLICATION_CONTROLLER_COUNT = 119 [(CounterOpts) = {Name: "ReplicationControllers"}]; + COUNTER_IN_FLIGHT_OPS_TxCreateReplication = 120 [(CounterOpts) = {Name: "InFlightOps/CreateReplication"}]; + COUNTER_IN_FLIGHT_OPS_TxAlterReplication = 121 [(CounterOpts) = {Name: "InFlightOps/AlterReplication"}]; + COUNTER_IN_FLIGHT_OPS_TxDropReplication = 122 [(CounterOpts) = {Name: "InFlightOps/DropReplication"}]; } enum ECumulativeCounters { @@ -203,8 +203,8 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxCreateFileStore = 46 [(CounterOpts) = {Name: "FinishedOps/CreateFileStore"}]; COUNTER_FINISHED_OPS_TxAlterFileStore = 47 [(CounterOpts) = {Name: "FinishedOps/AlterFileStore"}]; COUNTER_FINISHED_OPS_TxDropFileStore = 48 [(CounterOpts) = {Name: "FinishedOps/DropFileStore"}]; - - COUNTER_FINISHED_OPS_TxRestore = 49 [(CounterOpts) = {Name: "FinishedOps/Restore"}]; + + COUNTER_FINISHED_OPS_TxRestore = 49 [(CounterOpts) = {Name: "FinishedOps/Restore"}]; COUNTER_FINISHED_OPS_TxCreateOlapStore = 50 [(CounterOpts) = {Name: "FinishedOps/CreateOlapStore"}]; COUNTER_FINISHED_OPS_TxAlterOlapStore = 51 [(CounterOpts) = {Name: "FinishedOps/AlterOlapStore"}]; @@ -212,14 +212,14 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxCreateOlapTable = 53 [(CounterOpts) = {Name: "FinishedOps/CreateOlapTable"}]; COUNTER_FINISHED_OPS_TxAlterOlapTable = 54 [(CounterOpts) = {Name: "FinishedOps/AlterOlapTable"}]; COUNTER_FINISHED_OPS_TxDropOlapTable = 55 [(CounterOpts) = {Name: "FinishedOps/DropOlapTable"}]; - - COUNTER_FINISHED_OPS_TxCreateCdcStream = 56 [(CounterOpts) = {Name: "FinishedOps/CreateCdcStream"}]; - COUNTER_FINISHED_OPS_TxCreateCdcStreamAtTable = 57 [(CounterOpts) = {Name: "FinishedOps/CreateCdcStreamAtTable"}]; - COUNTER_FINISHED_OPS_TxAlterCdcStream = 58 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStream"}]; - COUNTER_FINISHED_OPS_TxAlterCdcStreamAtTable = 59 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStreamAtTable"}]; - COUNTER_FINISHED_OPS_TxDropCdcStream = 60 [(CounterOpts) = {Name: "FinishedOps/DropCdcStream"}]; - COUNTER_FINISHED_OPS_TxDropCdcStreamAtTable = 61 [(CounterOpts) = {Name: "FinishedOps/DropCdcStreamAtTable"}]; + + COUNTER_FINISHED_OPS_TxCreateCdcStream = 56 [(CounterOpts) = {Name: "FinishedOps/CreateCdcStream"}]; + COUNTER_FINISHED_OPS_TxCreateCdcStreamAtTable = 57 [(CounterOpts) = {Name: "FinishedOps/CreateCdcStreamAtTable"}]; + COUNTER_FINISHED_OPS_TxAlterCdcStream = 58 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStream"}]; + COUNTER_FINISHED_OPS_TxAlterCdcStreamAtTable = 59 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStreamAtTable"}]; + COUNTER_FINISHED_OPS_TxDropCdcStream = 60 [(CounterOpts) = {Name: "FinishedOps/DropCdcStream"}]; + COUNTER_FINISHED_OPS_TxDropCdcStreamAtTable = 61 [(CounterOpts) = {Name: "FinishedOps/DropCdcStreamAtTable"}]; COUNTER_FINISHED_OPS_TxMoveTable = 62 [(CounterOpts) = {Name: "FinishedOps/MoveTable"}]; COUNTER_FINISHED_OPS_TxMoveTableIndex = 63 [(CounterOpts) = {Name: "FinishedOps/MoveTableIndex"}]; @@ -231,10 +231,10 @@ enum ECumulativeCounters { COUNTER_BACKGROUND_COMPACTION_FAILED = 67 [(CounterOpts) = {Name: "BackgroundCompactionFailed"}]; COUNTER_BACKGROUND_COMPACTION_OK = 68 [(CounterOpts) = {Name: "BackgroundCompactionOK"}]; COUNTER_BACKGROUND_COMPACTION_TIMEOUT = 69 [(CounterOpts) = {Name: "BackgroundCompactionTimeout"}]; - - COUNTER_FINISHED_OPS_TxCreateReplication = 70 [(CounterOpts) = {Name: "FinishedOps/CreateReplication"}]; - COUNTER_FINISHED_OPS_TxAlterReplication = 71 [(CounterOpts) = {Name: "FinishedOps/AlterReplication"}]; - COUNTER_FINISHED_OPS_TxDropReplication = 72 [(CounterOpts) = {Name: "FinishedOps/DropReplication"}]; + + COUNTER_FINISHED_OPS_TxCreateReplication = 70 [(CounterOpts) = {Name: "FinishedOps/CreateReplication"}]; + COUNTER_FINISHED_OPS_TxAlterReplication = 71 [(CounterOpts) = {Name: "FinishedOps/AlterReplication"}]; + COUNTER_FINISHED_OPS_TxDropReplication = 72 [(CounterOpts) = {Name: "FinishedOps/DropReplication"}]; } enum EPercentileCounters { @@ -260,20 +260,20 @@ enum EPercentileCounters { }; COUNTER_BACKGROUND_COMPACTION_OK_LATENCY = 0 [(CounterOpts) = {Name: "BackgroundCompactionOkLatency"}]; - - COUNTER_NUM_SHARDS_BY_TTL_LAG = 1 [(CounterOpts) = { - Name: "NumShardsByTtlLag" - Integral: true - Ranges { Value: 0 Name: "0" } - Ranges { Value: 900 Name: "900" } - Ranges { Value: 1800 Name: "1800" } - Ranges { Value: 3600 Name: "3600" } - Ranges { Value: 7200 Name: "7200" } - Ranges { Value: 14400 Name: "14400" } - Ranges { Value: 28800 Name: "28800" } - Ranges { Value: 57600 Name: "57600" } - Ranges { Value: 86400 Name: "inf" } - }]; + + COUNTER_NUM_SHARDS_BY_TTL_LAG = 1 [(CounterOpts) = { + Name: "NumShardsByTtlLag" + Integral: true + Ranges { Value: 0 Name: "0" } + Ranges { Value: 900 Name: "900" } + Ranges { Value: 1800 Name: "1800" } + Ranges { Value: 3600 Name: "3600" } + Ranges { Value: 7200 Name: "7200" } + Ranges { Value: 14400 Name: "14400" } + Ranges { Value: 28800 Name: "28800" } + Ranges { Value: 57600 Name: "57600" } + Ranges { Value: 86400 Name: "inf" } + }]; } enum ETxTypes { @@ -291,23 +291,23 @@ enum ETxTypes { TXTYPE_INIT_SPLIT_DST_ACK = 11 [(TxTypeOpts) = {Name: "TxInitSplitDstAck"}]; TXTYPE_SPLIT_ACK = 12 [(TxTypeOpts) = {Name: "TxSplitAck"}]; TXTYPE_SPLIT_PARTITIONING_CHANGED_DST_ACK = 13 [(TxTypeOpts) = {Name: "TxSplitPartitioningChangedAck"}]; - TXTYPE_CANCEL_BACKUP_IMPL = 14 [(TxTypeOpts) = {Name: "TxCancelBackupImpl"}]; + TXTYPE_CANCEL_BACKUP_IMPL = 14 [(TxTypeOpts) = {Name: "TxCancelBackupImpl"}]; TXTYPE_MONITORING = 15 [(TxTypeOpts) = {Name: "TxMonitoring"}]; TXTYPE_SUBDOMAIN_CONFIGURE_RESULT = 16 [(TxTypeOpts) = {Name: "TTxSubDomainConfigureStatus"}]; TXTYPE_BLOCKSTORE_CONFIG_RESULT = 17 [(TxTypeOpts) = {Name: "TTxBlockStoreConfigResult"}]; TXTYPE_KESUS_CONFIG_RESULT = 18 [(TxTypeOpts) = {Name: "TTxKesusConfigResult"}]; TXTYPE_PROPOSE = 19 [(TxTypeOpts) = {Name: "TTxOperationPropose"}]; TXTYPE_DATASHARD_SCHEMA_CHANGED = 20 [(TxTypeOpts) = {Name: "TxDataShardSchemaChanged"}]; - TXTYPE_DEFERRED_UPDATE_ON_SCHEME_BOARD = 21 [(TxTypeOpts) = {Name: "TxDeferredUpdateOnSchemeBoard"}]; + TXTYPE_DEFERRED_UPDATE_ON_SCHEME_BOARD = 21 [(TxTypeOpts) = {Name: "TxDeferredUpdateOnSchemeBoard"}]; TXTYPE_CLEAN_TABLES = 22 [(TxTypeOpts) = {Name: "TxCleanTables"}]; - - TXTYPE_CREATE_EXPORT = 23 [(TxTypeOpts) = {Name: "TxCreateExport"}]; - TXTYPE_GET_EXPORT = 24 [(TxTypeOpts) = {Name: "TxGetExport"}]; - TXTYPE_CANCEL_EXPORT = 25 [(TxTypeOpts) = {Name: "TxCancelExport"}]; - TXTYPE_CANCEL_EXPORT_ACK = 26 [(TxTypeOpts) = {Name: "TxCancelExportAck"}]; - TXTYPE_FORGET_EXPORT = 27 [(TxTypeOpts) = {Name: "TxForgetExport"}]; - TXTYPE_LIST_EXPORTS = 28 [(TxTypeOpts) = {Name: "TxListExports"}]; - TXTYPE_EXPORT_PROGRESS = 29 [(TxTypeOpts) = {Name: "TxExportProgress"}]; + + TXTYPE_CREATE_EXPORT = 23 [(TxTypeOpts) = {Name: "TxCreateExport"}]; + TXTYPE_GET_EXPORT = 24 [(TxTypeOpts) = {Name: "TxGetExport"}]; + TXTYPE_CANCEL_EXPORT = 25 [(TxTypeOpts) = {Name: "TxCancelExport"}]; + TXTYPE_CANCEL_EXPORT_ACK = 26 [(TxTypeOpts) = {Name: "TxCancelExportAck"}]; + TXTYPE_FORGET_EXPORT = 27 [(TxTypeOpts) = {Name: "TxForgetExport"}]; + TXTYPE_LIST_EXPORTS = 28 [(TxTypeOpts) = {Name: "TxListExports"}]; + TXTYPE_EXPORT_PROGRESS = 29 [(TxTypeOpts) = {Name: "TxExportProgress"}]; TXTYPE_SYNC_TENANT = 30 [(TxTypeOpts) = {Name: "TxSyncTenant"}]; TXTYPE_UPDATE_TENANT = 31 [(TxTypeOpts) = {Name: "TxUpdateTenant"}]; @@ -323,23 +323,23 @@ enum ETxTypes { TXTYPE_FORGET_INDEX_BUILD = 40 [(TxTypeOpts) = {Name: "TxForgetIndexBuild"}]; TXTYPE_LIST_INDEX_BUILD = 41 [(TxTypeOpts) = {Name: "TxListIndexBuild"}]; TXTYPE_PROGRESS_INDEX_BUILD = 42 [(TxTypeOpts) = {Name: "TxProgressIndexBuild"}]; - - TXTYPE_SCHEDULE_CONDITIONAL_ERASE = 43 [(TxTypeOpts) = {Name: "TxScheduleConditionalErase"}]; - TXTYPE_RUN_CONDITIONAL_ERASE = 44 [(TxTypeOpts) = {Name: "TxRunConditionalErase"}]; - - TXTYPE_STORE_PARTITION_STATS = 45 [(TxTypeOpts) = {Name: "TxStorePartitionStats"}]; + + TXTYPE_SCHEDULE_CONDITIONAL_ERASE = 43 [(TxTypeOpts) = {Name: "TxScheduleConditionalErase"}]; + TXTYPE_RUN_CONDITIONAL_ERASE = 44 [(TxTypeOpts) = {Name: "TxRunConditionalErase"}]; + + TXTYPE_STORE_PARTITION_STATS = 45 [(TxTypeOpts) = {Name: "TxStorePartitionStats"}]; TXTYPE_FILESTORE_CONFIG_RESULT = 46 [(TxTypeOpts) = {Name: "TxFileStoreConfigResult"}]; TXTYPE_MAKEBILL_INDEX_BUILD = 47 [(TxTypeOpts) = {Name: "TxBillingIndexBuild"}]; - - TXTYPE_CREATE_IMPORT = 48 [(TxTypeOpts) = {Name: "TxCreateImport"}]; - TXTYPE_GET_IMPORT = 49 [(TxTypeOpts) = {Name: "TxGetImport"}]; - TXTYPE_CANCEL_IMPORT = 50 [(TxTypeOpts) = {Name: "TxCancelImport"}]; - TXTYPE_CANCEL_IMPORT_ACK = 51 [(TxTypeOpts) = {Name: "TxCancelImportAck"}]; - TXTYPE_FORGET_IMPORT = 52 [(TxTypeOpts) = {Name: "TxForgetImport"}]; - TXTYPE_LIST_IMPORTS = 53 [(TxTypeOpts) = {Name: "TxListImports"}]; - TXTYPE_IMPORT_PROGRESS = 54 [(TxTypeOpts) = {Name: "TxImportProgress"}]; + + TXTYPE_CREATE_IMPORT = 48 [(TxTypeOpts) = {Name: "TxCreateImport"}]; + TXTYPE_GET_IMPORT = 49 [(TxTypeOpts) = {Name: "TxGetImport"}]; + TXTYPE_CANCEL_IMPORT = 50 [(TxTypeOpts) = {Name: "TxCancelImport"}]; + TXTYPE_CANCEL_IMPORT_ACK = 51 [(TxTypeOpts) = {Name: "TxCancelImportAck"}]; + TXTYPE_FORGET_IMPORT = 52 [(TxTypeOpts) = {Name: "TxForgetImport"}]; + TXTYPE_LIST_IMPORTS = 53 [(TxTypeOpts) = {Name: "TxListImports"}]; + TXTYPE_IMPORT_PROGRESS = 54 [(TxTypeOpts) = {Name: "TxImportProgress"}]; TXTYPE_SERVERLESS_STORAGE_BILLING = 55 [(TxTypeOpts) = {Name: "TxServerlessStorageBilling"}]; TXTYPE_OLAPSHARD_PROPOSE_RESULT = 56 [(TxTypeOpts) = {Name: "TxOlapShardProposeResult"}]; TXTYPE_COLUMNSHARD_PROPOSE_RESULT = 57 [(TxTypeOpts) = {Name: "TxColumnShardProposeResult"}]; @@ -365,8 +365,8 @@ enum ETxTypes { TXTYPE_NOTIFY_OPERATION_COMPLETE_BARRIER = 72 [(TxTypeOpts) = {Name: "TxNotifyOperationCompleteBarrier"}]; TXTYPE_PARTITION_HISTOGRAM = 73 [(TxTypeOpts) = {Name: "TxPartitionHistogram"}]; - - TXTYPE_CREATE_REPLICATION_RESULT = 74 [(TxTypeOpts) = {Name: "TxCreateReplicationResult"}]; - TXTYPE_ALTER_REPLICATION_RESULT = 75 [(TxTypeOpts) = {Name: "TxAlterReplicationResult"}]; - TXTYPE_DROP_REPLICATION_RESULT = 76 [(TxTypeOpts) = {Name: "TxDropReplicationResult"}]; + + TXTYPE_CREATE_REPLICATION_RESULT = 74 [(TxTypeOpts) = {Name: "TxCreateReplicationResult"}]; + TXTYPE_ALTER_REPLICATION_RESULT = 75 [(TxTypeOpts) = {Name: "TxAlterReplicationResult"}]; + TXTYPE_DROP_REPLICATION_RESULT = 76 [(TxTypeOpts) = {Name: "TxDropReplicationResult"}]; } diff --git a/ydb/core/protos/export.proto b/ydb/core/protos/export.proto index 250f0e54f52..929da802c89 100644 --- a/ydb/core/protos/export.proto +++ b/ydb/core/protos/export.proto @@ -2,121 +2,121 @@ import "ydb/public/api/protos/ydb_export.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; - -package NKikimrExport; -option java_package = "ru.yandex.kikimr.proto"; - -message TExport { - optional uint64 Id = 1; - optional Ydb.StatusIds.StatusCode Status = 2; - repeated Ydb.Issue.IssueMessage Issues = 3; - optional Ydb.Export.ExportProgress.Progress Progress = 4; - repeated Ydb.Export.ExportItemProgress ItemsProgress = 7; - oneof Settings { - Ydb.Export.ExportToYtSettings ExportToYtSettings = 5; - Ydb.Export.ExportToS3Settings ExportToS3Settings = 6; - } -} - -message TCreateExportRequest { - optional Ydb.Operations.OperationParams OperationParams = 1; - oneof Settings { - Ydb.Export.ExportToYtSettings ExportToYtSettings = 2; - Ydb.Export.ExportToS3Settings ExportToS3Settings = 3; - } -} - -message TEvCreateExportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 2; - optional string UserSID = 4; - optional TCreateExportRequest Request = 3; -} - -message TCreateExportResponse { - optional TExport Entry = 1; -} - -message TEvCreateExportResponse { - optional uint64 TxId = 1; - optional TCreateExportResponse Response = 2; -} - -message TGetExportRequest { - optional uint64 Id = 1; -} - -message TEvGetExportRequest { - optional string DatabaseName = 2; - optional TGetExportRequest Request = 1; -} - -message TGetExportResponse { - optional TExport Entry = 1; -} - -message TEvGetExportResponse { - optional TGetExportResponse Response = 1; -} - -message TCancelExportRequest { - optional uint64 Id = 1; -} - -message TEvCancelExportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 3; - optional TCancelExportRequest Request = 2; -} - -message TCancelExportResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; -} - -message TEvCancelExportResponse { - optional uint64 TxId = 1; - optional TCancelExportResponse Response = 2; -} - -message TForgetExportRequest { - optional uint64 Id = 1; -} - -message TEvForgetExportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 3; - optional TForgetExportRequest Request = 2; -} - -message TForgetExportResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; -} - -message TEvForgetExportResponse { - optional uint64 TxId = 1; - optional TForgetExportResponse Response = 2; -} - -message TListExportsRequest { - optional uint64 PageSize = 1; - optional string PageToken = 2; - optional string Kind = 3; -} - -message TEvListExportsRequest { - optional string DatabaseName = 2; - optional TListExportsRequest Request = 1; -} - -message TListExportsResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; - repeated TExport Entries = 3; - optional string NextPageToken = 4; -} - -message TEvListExportsResponse { - optional TListExportsResponse Response = 1; -} + +package NKikimrExport; +option java_package = "ru.yandex.kikimr.proto"; + +message TExport { + optional uint64 Id = 1; + optional Ydb.StatusIds.StatusCode Status = 2; + repeated Ydb.Issue.IssueMessage Issues = 3; + optional Ydb.Export.ExportProgress.Progress Progress = 4; + repeated Ydb.Export.ExportItemProgress ItemsProgress = 7; + oneof Settings { + Ydb.Export.ExportToYtSettings ExportToYtSettings = 5; + Ydb.Export.ExportToS3Settings ExportToS3Settings = 6; + } +} + +message TCreateExportRequest { + optional Ydb.Operations.OperationParams OperationParams = 1; + oneof Settings { + Ydb.Export.ExportToYtSettings ExportToYtSettings = 2; + Ydb.Export.ExportToS3Settings ExportToS3Settings = 3; + } +} + +message TEvCreateExportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 2; + optional string UserSID = 4; + optional TCreateExportRequest Request = 3; +} + +message TCreateExportResponse { + optional TExport Entry = 1; +} + +message TEvCreateExportResponse { + optional uint64 TxId = 1; + optional TCreateExportResponse Response = 2; +} + +message TGetExportRequest { + optional uint64 Id = 1; +} + +message TEvGetExportRequest { + optional string DatabaseName = 2; + optional TGetExportRequest Request = 1; +} + +message TGetExportResponse { + optional TExport Entry = 1; +} + +message TEvGetExportResponse { + optional TGetExportResponse Response = 1; +} + +message TCancelExportRequest { + optional uint64 Id = 1; +} + +message TEvCancelExportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 3; + optional TCancelExportRequest Request = 2; +} + +message TCancelExportResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; +} + +message TEvCancelExportResponse { + optional uint64 TxId = 1; + optional TCancelExportResponse Response = 2; +} + +message TForgetExportRequest { + optional uint64 Id = 1; +} + +message TEvForgetExportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 3; + optional TForgetExportRequest Request = 2; +} + +message TForgetExportResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; +} + +message TEvForgetExportResponse { + optional uint64 TxId = 1; + optional TForgetExportResponse Response = 2; +} + +message TListExportsRequest { + optional uint64 PageSize = 1; + optional string PageToken = 2; + optional string Kind = 3; +} + +message TEvListExportsRequest { + optional string DatabaseName = 2; + optional TListExportsRequest Request = 1; +} + +message TListExportsResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; + repeated TExport Entries = 3; + optional string NextPageToken = 4; +} + +message TEvListExportsResponse { + optional TListExportsResponse Response = 1; +} diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 3fcd75289e9..33e598c1c25 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1,7 +1,7 @@ -import "ydb/core/protos/base.proto"; +import "ydb/core/protos/base.proto"; import "ydb/core/protos/bind_channel_storage_pool.proto"; import "ydb/core/protos/pqconfig.proto"; -import "ydb/core/protos/replication.proto"; +import "ydb/core/protos/replication.proto"; import "ydb/core/protos/table_stats.proto"; import "ydb/core/protos/tablet.proto"; import "ydb/core/protos/subdomains.proto"; @@ -285,40 +285,40 @@ message TShardIdx { optional uint64 LocalId = 2; } -message TTTLSettings { - enum EUnit { - UNIT_AUTO = 0; - UNIT_SECONDS = 1; - UNIT_MILLISECONDS = 2; - UNIT_MICROSECONDS = 3; - UNIT_NANOSECONDS = 4; - } - - message TSysSettings { - optional uint64 RunInterval = 1 [default = 3600000000]; // 1 hour - optional uint64 RetryInterval = 2 [default = 300000000]; // 5 minutes - optional uint32 BatchMaxBytes = 3 [default = 512000]; - optional uint32 BatchMinKeys = 4 [default = 1]; - optional uint32 BatchMaxKeys = 5 [default = 256]; - optional uint32 MaxShardsInFlight = 6 [default = 1]; // zero means no limit - } - - message TEnabled { - optional string ColumnName = 1; +message TTTLSettings { + enum EUnit { + UNIT_AUTO = 0; + UNIT_SECONDS = 1; + UNIT_MILLISECONDS = 2; + UNIT_MICROSECONDS = 3; + UNIT_NANOSECONDS = 4; + } + + message TSysSettings { + optional uint64 RunInterval = 1 [default = 3600000000]; // 1 hour + optional uint64 RetryInterval = 2 [default = 300000000]; // 5 minutes + optional uint32 BatchMaxBytes = 3 [default = 512000]; + optional uint32 BatchMinKeys = 4 [default = 1]; + optional uint32 BatchMaxKeys = 5 [default = 256]; + optional uint32 MaxShardsInFlight = 6 [default = 1]; // zero means no limit + } + + message TEnabled { + optional string ColumnName = 1; optional uint32 ExpireAfterSeconds = 2; - optional EUnit ColumnUnit = 3; - optional TSysSettings SysSettings = 4; - } - - message TDisabled { - } - - oneof Status { - TEnabled Enabled = 1; - TDisabled Disabled = 2; - } -} - + optional EUnit ColumnUnit = 3; + optional TSysSettings SysSettings = 4; + } + + message TDisabled { + } + + oneof Status { + TEnabled Enabled = 1; + TDisabled Disabled = 2; + } +} + message TTableDescription { optional string Name = 1; optional uint64 Id_Deprecated = 2; // LocalPathId, deprecated @@ -345,14 +345,14 @@ message TTableDescription { optional uint64 TableSchemaVersion = 33; - optional NKikimrProto.TPathID PathId = 34; // TPathId, owerride Id = 2 - - optional TTTLSettings TTLSettings = 35; - + optional NKikimrProto.TPathID PathId = 34; // TPathId, owerride Id = 2 + + optional TTTLSettings TTLSettings = 35; + optional bool OmitFollowers = 36 [default = false]; // used with CopyFromTable - optional bool IsBackup = 37 [default = false]; - - repeated TCdcStreamDescription CdcStreams = 38; + optional bool IsBackup = 37 [default = false]; + + repeated TCdcStreamDescription CdcStreams = 38; repeated TSequenceDescription Sequences = 39; } @@ -695,55 +695,55 @@ message TAlterLogin { } } -enum ECdcStreamState { - ECdcStreamStateInvalid = 0; - ECdcStreamStateReady = 1; - ECdcStreamStateDisabled = 2; -} - -enum ECdcStreamMode { - ECdcStreamModeInvalid = 0; - ECdcStreamModeKeysOnly = 1; - ECdcStreamModeUpdate = 2; - ECdcStreamModeNewImage = 3; - ECdcStreamModeOldImage = 4; - ECdcStreamModeNewAndOldImages = 5; -} - -message TCdcStreamDescription { - optional string Name = 1; - optional ECdcStreamMode Mode = 2; - optional NKikimrProto.TPathID PathId = 3; - optional ECdcStreamState State = 4; - optional uint64 SchemaVersion = 5; -} - -message TCreateCdcStream { - optional string TableName = 1; - optional TCdcStreamDescription StreamDescription = 2; -} - -message TAlterCdcStream { - optional string TableName = 1; - optional string StreamName = 2; - - message TDisable { - } - - oneof Action { - TDisable Disable = 3; - } -} - -message TDropCdcStream { - optional string TableName = 1; - optional string StreamName = 2; -} - +enum ECdcStreamState { + ECdcStreamStateInvalid = 0; + ECdcStreamStateReady = 1; + ECdcStreamStateDisabled = 2; +} + +enum ECdcStreamMode { + ECdcStreamModeInvalid = 0; + ECdcStreamModeKeysOnly = 1; + ECdcStreamModeUpdate = 2; + ECdcStreamModeNewImage = 3; + ECdcStreamModeOldImage = 4; + ECdcStreamModeNewAndOldImages = 5; +} + +message TCdcStreamDescription { + optional string Name = 1; + optional ECdcStreamMode Mode = 2; + optional NKikimrProto.TPathID PathId = 3; + optional ECdcStreamState State = 4; + optional uint64 SchemaVersion = 5; +} + +message TCreateCdcStream { + optional string TableName = 1; + optional TCdcStreamDescription StreamDescription = 2; +} + +message TAlterCdcStream { + optional string TableName = 1; + optional string StreamName = 2; + + message TDisable { + } + + oneof Action { + TDisable Disable = 3; + } +} + +message TDropCdcStream { + optional string TableName = 1; + optional string StreamName = 2; +} + enum EIndexType { EIndexTypeInvalid = 0; EIndexTypeGlobal = 1; - EIndexTypeGlobalAsync = 2; + EIndexTypeGlobalAsync = 2; } enum EIndexState { @@ -767,8 +767,8 @@ message TIndexDescription { optional uint64 PathOwnerId = 7; repeated string DataColumnNames = 8; - // DataSize + IndexSize of indexImplTable - optional uint64 DataSize = 9; + // DataSize + IndexSize of indexImplTable + optional uint64 DataSize = 9; } message TIndexCreationConfig { @@ -807,91 +807,91 @@ message TCopyTableConfig { //TTableDescription implemets copying a table in orig optional bool OmitIndexes = 3 [default = false]; optional bool OmitFollowers = 4 [default = false]; - optional bool IsBackup = 5 [default = false]; + optional bool IsBackup = 5 [default = false]; } message TConsistentTableCopyingConfig { repeated TCopyTableConfig CopyTableDescriptions = 1; } -message TYTSettings { - optional string Host = 1; - optional string TablePattern = 2; +message TYTSettings { + optional string Host = 1; + optional string TablePattern = 2; optional string Token = 3; - optional uint32 Port = 4 [default = 80]; - - reserved 5; // YSONasText - reserved 6; // AddScheme - reserved 7; // CreateYTTable - - optional bool UseTypeV3 = 8; + optional uint32 Port = 4 [default = 80]; + + reserved 5; // YSONasText + reserved 6; // AddScheme + reserved 7; // CreateYTTable + + optional bool UseTypeV3 = 8; +}; + +message TS3Settings { + enum EScheme { + HTTP = 0; + HTTPS = 1; + } + + optional string Endpoint = 1; + optional EScheme Scheme = 2 [default = HTTPS]; + optional string Bucket = 3; + optional string ObjectKeyPattern = 4; // dst for backup, src for restore + optional string AccessKey = 5; + optional string SecretKey = 6; + optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7; + + message TLimits { + optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB + optional uint32 WriteBatchSize = 2 [default = 67108864]; // 64 MB + }; + + optional TLimits Limits = 100; }; -message TS3Settings { - enum EScheme { - HTTP = 0; - HTTPS = 1; - } - - optional string Endpoint = 1; - optional EScheme Scheme = 2 [default = HTTPS]; - optional string Bucket = 3; - optional string ObjectKeyPattern = 4; // dst for backup, src for restore - optional string AccessKey = 5; - optional string SecretKey = 6; - optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7; - - message TLimits { - optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB - optional uint32 WriteBatchSize = 2 [default = 67108864]; // 64 MB - }; - - optional TLimits Limits = 100; -}; - message TBackupTask { optional string TableName = 1; - optional uint64 TableId = 2; + optional uint64 TableId = 2; optional uint32 ShardNum = 3; optional uint32 NumberOfRetries = 5; - - reserved 6; // IfFailedBeforeTxId - reserved 7; // UserProvidedId - reserved 8; // UserProvidedComment - - oneof Settings { - TYTSettings YTSettings = 4; - TS3Settings S3Settings = 9; - } - - optional TPathDescription Table = 10; // for further restore - - message TScanSettings { - optional uint64 RowsBatchSize = 1 [default = 0]; // no limit - optional uint64 BytesBatchSize = 2 [default = 67108864]; // 64 MB - } - - optional TScanSettings ScanSettings = 11; - optional bool NeedToBill = 12; -} - -message TRestoreTask { - optional string TableName = 1; - optional uint64 TableId = 2; - optional TTableDescription TableDescription = 3; - optional uint32 ShardNum = 4; - optional uint32 NumberOfRetries = 5; - - oneof Settings { - TS3Settings S3Settings = 6; - } -} - + + reserved 6; // IfFailedBeforeTxId + reserved 7; // UserProvidedId + reserved 8; // UserProvidedComment + + oneof Settings { + TYTSettings YTSettings = 4; + TS3Settings S3Settings = 9; + } + + optional TPathDescription Table = 10; // for further restore + + message TScanSettings { + optional uint64 RowsBatchSize = 1 [default = 0]; // no limit + optional uint64 BytesBatchSize = 2 [default = 67108864]; // 64 MB + } + + optional TScanSettings ScanSettings = 11; + optional bool NeedToBill = 12; +} + +message TRestoreTask { + optional string TableName = 1; + optional uint64 TableId = 2; + optional TTableDescription TableDescription = 3; + optional uint32 ShardNum = 4; + optional uint32 NumberOfRetries = 5; + + oneof Settings { + TS3Settings S3Settings = 6; + } +} + message TPersQueueGroupDescription { message TPartition { optional uint32 PartitionId = 1; optional uint64 TabletId = 2; - optional NKikimrPQ.TPartitionKeyRange KeyRange = 3; + optional NKikimrPQ.TPartitionKeyRange KeyRange = 3; } optional string Name = 1; // mandatory @@ -909,14 +909,14 @@ message TPersQueueGroupDescription { repeated TPartition Partitions = 6; // do not set optional uint64 AlterVersion = 7; optional uint64 BalancerTabletID = 8; - - // Can be passed upon creation. - // PQTabletConfig.PartitionKeySchema must be set. - // Size must be equal to TotalGroupCount - 1 - repeated NKikimrMiniKQL.TValue PartitionBoundaries = 12; - - // Can be passed upon creation - optional NKikimrPQ.TBootstrapConfig BootstrapConfig = 13; + + // Can be passed upon creation. + // PQTabletConfig.PartitionKeySchema must be set. + // Size must be equal to TotalGroupCount - 1 + repeated NKikimrMiniKQL.TValue PartitionBoundaries = 12; + + // Can be passed upon creation + optional NKikimrPQ.TBootstrapConfig BootstrapConfig = 13; } message TRtmrVolumeDescription { @@ -969,7 +969,7 @@ message TSolomonVolumeDescription { optional uint64 PartitionId = 1; optional uint64 TabletId = 2; optional uint64 ShardIdx = 3; - repeated NKikimrStoragePool.TChannelBind BoundChannels = 4; + repeated NKikimrStoragePool.TChannelBind BoundChannels = 4; } optional string Name = 1; // mandatory @@ -998,8 +998,8 @@ message TAlterSolomonVolume { optional uint32 ChannelProfileId = 2; optional uint64 PartitionCount = 3; - - optional bool UpdateChannelsBinding = 4 [default = false]; + + optional bool UpdateChannelsBinding = 4 [default = false]; } message TBlockStoreAssignOp { @@ -1041,7 +1041,7 @@ message TMove { // private description of the operation message TSequenceDescription { optional string Name = 1; // mandatory - optional NKikimrProto.TPathID PathId = 2; // sequence path id, assigned by schemeshard + optional NKikimrProto.TPathID PathId = 2; // sequence path id, assigned by schemeshard optional uint64 Version = 3; // incremented every time sequence is altered optional uint64 SequenceShard = 4; // current sequenceshard, assigned by schemeshard @@ -1058,14 +1058,14 @@ message TSequenceSharding { repeated TShardIdx SequenceShards = 1; } -message TReplicationDescription { - optional string Name = 1; - optional NKikimrReplication.TReplicationConfig Config = 2; - optional NKikimrProto.TPathID PathId = 3; // path id assigned by schemeshard - optional uint64 Version = 4; - optional uint64 ControllerId = 5; // replication controller's tablet id -} - +message TReplicationDescription { + optional string Name = 1; + optional NKikimrReplication.TReplicationConfig Config = 2; + optional NKikimrProto.TPathID PathId = 3; // path id assigned by schemeshard + optional uint64 Version = 4; + optional uint64 ControllerId = 5; // replication controller's tablet id +} + enum EOperationType { ESchemeOpMkDir = 1; ESchemeOpCreateTable = 2; @@ -1134,8 +1134,8 @@ enum EOperationType { ESchemeOpCreateFileStore = 51; ESchemeOpAlterFileStore = 52; ESchemeOpDropFileStore = 53; - - ESchemeOpRestore = 54; + + ESchemeOpRestore = 54; ESchemeOpCreateColumnStore = 55; ESchemeOpAlterColumnStore = 56; @@ -1145,20 +1145,20 @@ enum EOperationType { ESchemeOpDropColumnTable = 60; ESchemeOpAlterLogin = 61; - - /// CDC - // Create - ESchemeOpCreateCdcStream = 62; - ESchemeOpCreateCdcStreamImpl = 63; - ESchemeOpCreateCdcStreamAtTable = 64; - // Alter - ESchemeOpAlterCdcStream = 65; - ESchemeOpAlterCdcStreamImpl = 66; - ESchemeOpAlterCdcStreamAtTable = 67; - // Drop - ESchemeOpDropCdcStream = 68; - ESchemeOpDropCdcStreamImpl = 69; - ESchemeOpDropCdcStreamAtTable = 70; + + /// CDC + // Create + ESchemeOpCreateCdcStream = 62; + ESchemeOpCreateCdcStreamImpl = 63; + ESchemeOpCreateCdcStreamAtTable = 64; + // Alter + ESchemeOpAlterCdcStream = 65; + ESchemeOpAlterCdcStreamImpl = 66; + ESchemeOpAlterCdcStreamAtTable = 67; + // Drop + ESchemeOpDropCdcStream = 68; + ESchemeOpDropCdcStreamImpl = 69; + ESchemeOpDropCdcStreamAtTable = 70; ESchemeOpMoveTable = 71; ESchemeOpMoveTableIndex = 72; @@ -1167,11 +1167,11 @@ enum EOperationType { ESchemeOpCreateSequence = 73; ESchemeOpAlterSequence = 74; ESchemeOpDropSequence = 75; - - // Replication - ESchemeOpCreateReplication = 76; - ESchemeOpAlterReplication = 77; - ESchemeOpDropReplication = 78; + + // Replication + ESchemeOpCreateReplication = 76; + ESchemeOpAlterReplication = 77; + ESchemeOpDropReplication = 78; } message TApplyIf { @@ -1260,19 +1260,19 @@ message TModifyScheme { optional TIndexBuildControl CancelIndexBuild = 35; optional TFileStoreDescription CreateFileStore = 37; optional TFileStoreDescription AlterFileStore = 38; - optional TRestoreTask Restore = 39; + optional TRestoreTask Restore = 39; optional TColumnStoreDescription CreateColumnStore = 40; optional TColumnTableDescription CreateColumnTable = 41; optional TAlterColumnStore AlterColumnStore = 42; optional TAlterColumnTable AlterColumnTable = 43; optional TAlterLogin AlterLogin = 44; - optional TCreateCdcStream CreateCdcStream = 45; - optional TAlterCdcStream AlterCdcStream = 46; - optional TDropCdcStream DropCdcStream = 47; + optional TCreateCdcStream CreateCdcStream = 45; + optional TAlterCdcStream AlterCdcStream = 46; + optional TDropCdcStream DropCdcStream = 47; optional TMove MoveTable = 48; optional TMove MoveTableIndex = 49; optional TSequenceDescription Sequence = 51; - optional TReplicationDescription Replication = 52; + optional TReplicationDescription Replication = 52; } // "Script", used by client to parse text files with multiple DDL commands @@ -1288,7 +1288,7 @@ message TDescribeOptions { optional bool ReturnChildren = 5 [default = true]; optional bool ReturnBoundaries = 6 [default = false]; optional bool ShowPrivateTable = 7 [default = false]; - optional bool ReturnChannelsBinding = 8 [default = false]; + optional bool ReturnChannelsBinding = 8 [default = false]; } // Request to read scheme for a specific path @@ -1323,18 +1323,18 @@ enum EPathType { EPathTypeFileStore = 11; EPathTypeColumnStore = 12; EPathTypeColumnTable = 13; - EPathTypeCdcStream = 14; + EPathTypeCdcStream = 14; EPathTypeSequence = 15; - EPathTypeReplication = 16; + EPathTypeReplication = 16; +} + +enum EPathSubType { + EPathSubTypeEmpty = 0; + EPathSubTypeSyncIndexImplTable = 1; + EPathSubTypeAsyncIndexImplTable = 2; + EPathSubTypeStreamImpl = 3; } -enum EPathSubType { - EPathSubTypeEmpty = 0; - EPathSubTypeSyncIndexImplTable = 1; - EPathSubTypeAsyncIndexImplTable = 2; - EPathSubTypeStreamImpl = 3; -} - enum EPathState { EPathStateNotExist = 1; EPathStateNoChanges = 2; @@ -1345,7 +1345,7 @@ enum EPathState { EPathStateBackup = 7; EPathStateUpgrade = 8; EPathStateMigrated = 9; - EPathStateRestore = 10; + EPathStateRestore = 10; EPathStateMoving = 11; } @@ -1372,10 +1372,10 @@ message TPathVersion { optional uint64 SubDomainStateVersion = 19; optional uint64 ColumnTableSchemaVersion = 20; optional uint64 ColumnTableTtlSettingsVersion = 21; - optional uint64 CdcStreamVersion = 22; + optional uint64 CdcStreamVersion = 22; optional uint64 SecurityStateVersion = 23; optional uint64 SequenceVersion = 24; - optional uint64 ReplicationVersion = 25; + optional uint64 ReplicationVersion = 25; } // Describes single path @@ -1393,7 +1393,7 @@ message TDirEntry { optional bytes ACL = 11; optional bytes EffectiveACL = 12; optional uint64 PathVersion = 13; - optional EPathSubType PathSubType = 14; + optional EPathSubType PathSubType = 14; optional TPathVersion Version = 15; optional uint64 BalancerTabletID = 999; //temporary optimization for old PQ read/write protocol. Must be removed later @@ -1418,7 +1418,7 @@ message TBackupProgress { optional uint32 ErrorCount = 3; optional uint64 StartTime = 4; repeated TShardError Errors = 5; - optional TYTSettings YTSettings = 6; + optional TYTSettings YTSettings = 6; optional uint64 DataTotalSize = 7; optional uint64 TxId = 8; } @@ -1428,7 +1428,7 @@ message TLastBackupResult { optional uint64 CompleteTimeStamp = 2; optional uint64 StartTimeStamp = 3; repeated TShardError Errors = 4; - optional TYTSettings YTSettings = 5; + optional TYTSettings YTSettings = 5; optional uint64 DataTotalSize = 6; optional uint64 TxId = 7; }; @@ -1457,9 +1457,9 @@ message TPathDescription { optional TFileStoreDescription FileStoreDescription = 20; optional TColumnStoreDescription ColumnStoreDescription = 21; optional TColumnTableDescription ColumnTableDescription = 22; - optional TCdcStreamDescription CdcStreamDescription = 23; + optional TCdcStreamDescription CdcStreamDescription = 23; optional TSequenceDescription SequenceDescription = 24; - optional TReplicationDescription ReplicationDescription = 25; + optional TReplicationDescription ReplicationDescription = 25; } // For persisting AlterTable Tx description in Schemeshard internal DB diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 51cd807253f..dc06523e5cc 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -29,8 +29,8 @@ enum EStatus { StatusInvalidParameter = 7; StatusMultipleModifications = 8; StatusReadOnly = 9; - StatusTxIdNotExists = 10; - StatusTxIsNotCancellable = 11; + StatusTxIdNotExists = 10; + StatusTxIsNotCancellable = 11; StatusAccessDenied = 12; StatusNotAvailable = 13; StatusPreconditionFailed = 14; @@ -77,17 +77,17 @@ message TEvDescribeSchemeResult { optional fixed64 PathOwnerId = 10; } -message TEvCancelTx { - optional uint64 TargetTxId = 1; +message TEvCancelTx { + optional uint64 TargetTxId = 1; optional string Path = 2; optional string SecurityToken = 3; optional uint64 TxId = 4; } -message TEvCancelTxResult { +message TEvCancelTxResult { optional EStatus Status = 1; optional string Result = 2; - optional uint64 TargetTxId = 3; + optional uint64 TargetTxId = 3; optional uint64 TxId = 4; } @@ -163,10 +163,10 @@ message TEvInitTenantSchemeShard { optional TSchemeLimits SchemeLimits = 11; optional bool InitiateMigration = 12; - - optional fixed64 ResourcesDomainOwnerId = 13; - optional fixed64 ResourcesDomainPathId = 14; - optional fixed64 SharedHive = 15; + + optional fixed64 ResourcesDomainOwnerId = 13; + optional fixed64 ResourcesDomainPathId = 14; + optional fixed64 SharedHive = 15; optional NKikimrSubDomains.TSchemeQuotas DeclaredSchemeQuotas = 16; optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 17; @@ -192,8 +192,8 @@ message TMigrateShard { } message TMigratePath { - optional NKikimrProto.TPathID PathId = 1; - optional NKikimrProto.TPathID ParentPathId = 2; + optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID ParentPathId = 2; optional string Name = 3; optional uint32 PathType = 4; optional uint64 StepCreated= 5; @@ -227,7 +227,7 @@ message TMigratePartition { } message TMigrateTable { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional uint32 NextColId = 2; optional string PartitionConfig = 3; optional uint32 AlterVersion = 4; @@ -238,7 +238,7 @@ message TMigrateTable { } message TMigrateTableIndex { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional uint64 AlterVersion = 2; optional uint32 Type = 3; optional uint32 State = 4; @@ -246,7 +246,7 @@ message TMigrateTableIndex { } message TMigrateKesus { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional uint64 Version = 2; optional string Config = 3; } @@ -266,7 +266,7 @@ message TEvMigrate { message TEvMigrateResult { optional fixed64 TenantSchemeShard = 1; optional uint64 SchemeShardGeneration = 2; - optional NKikimrProto.TPathID PathId = 3; + optional NKikimrProto.TPathID PathId = 3; repeated TShardID ShardIds = 4; } diff --git a/ydb/core/protos/import.proto b/ydb/core/protos/import.proto index 88cc76030ed..7ce4d497440 100644 --- a/ydb/core/protos/import.proto +++ b/ydb/core/protos/import.proto @@ -2,119 +2,119 @@ import "ydb/public/api/protos/ydb_import.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; - -package NKikimrImport; -option java_package = "ru.yandex.kikimr.proto"; - -message TImport { - optional uint64 Id = 1; - optional Ydb.StatusIds.StatusCode Status = 2; - repeated Ydb.Issue.IssueMessage Issues = 3; - optional Ydb.Import.ImportProgress.Progress Progress = 4; - repeated Ydb.Import.ImportItemProgress ItemsProgress = 5; - oneof Settings { - Ydb.Import.ImportFromS3Settings ImportFromS3Settings = 6; - } -} - -message TCreateImportRequest { - optional Ydb.Operations.OperationParams OperationParams = 1; - oneof Settings { - Ydb.Import.ImportFromS3Settings ImportFromS3Settings = 2; - } -} - -message TEvCreateImportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 2; - optional string UserSID = 3; - optional TCreateImportRequest Request = 4; -} - -message TCreateImportResponse { - optional TImport Entry = 1; -} - -message TEvCreateImportResponse { - optional uint64 TxId = 1; - optional TCreateImportResponse Response = 2; -} - -message TGetImportRequest { - optional uint64 Id = 1; -} - -message TEvGetImportRequest { - optional string DatabaseName = 2; - optional TGetImportRequest Request = 1; -} - -message TGetImportResponse { - optional TImport Entry = 1; -} - -message TEvGetImportResponse { - optional TGetImportResponse Response = 1; -} - -message TCancelImportRequest { - optional uint64 Id = 1; -} - -message TEvCancelImportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 3; - optional TCancelImportRequest Request = 2; -} - -message TCancelImportResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; -} - -message TEvCancelImportResponse { - optional uint64 TxId = 1; - optional TCancelImportResponse Response = 2; -} - -message TForgetImportRequest { - optional uint64 Id = 1; -} - -message TEvForgetImportRequest { - optional uint64 TxId = 1; - optional string DatabaseName = 3; - optional TForgetImportRequest Request = 2; -} - -message TForgetImportResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; -} - -message TEvForgetImportResponse { - optional uint64 TxId = 1; - optional TForgetImportResponse Response = 2; -} - -message TListImportsRequest { - optional uint64 PageSize = 1; - optional string PageToken = 2; - optional string Kind = 3; -} - -message TEvListImportsRequest { - optional string DatabaseName = 2; - optional TListImportsRequest Request = 1; -} - -message TListImportsResponse { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; - repeated TImport Entries = 3; - optional string NextPageToken = 4; -} - -message TEvListImportsResponse { - optional TListImportsResponse Response = 1; -} + +package NKikimrImport; +option java_package = "ru.yandex.kikimr.proto"; + +message TImport { + optional uint64 Id = 1; + optional Ydb.StatusIds.StatusCode Status = 2; + repeated Ydb.Issue.IssueMessage Issues = 3; + optional Ydb.Import.ImportProgress.Progress Progress = 4; + repeated Ydb.Import.ImportItemProgress ItemsProgress = 5; + oneof Settings { + Ydb.Import.ImportFromS3Settings ImportFromS3Settings = 6; + } +} + +message TCreateImportRequest { + optional Ydb.Operations.OperationParams OperationParams = 1; + oneof Settings { + Ydb.Import.ImportFromS3Settings ImportFromS3Settings = 2; + } +} + +message TEvCreateImportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 2; + optional string UserSID = 3; + optional TCreateImportRequest Request = 4; +} + +message TCreateImportResponse { + optional TImport Entry = 1; +} + +message TEvCreateImportResponse { + optional uint64 TxId = 1; + optional TCreateImportResponse Response = 2; +} + +message TGetImportRequest { + optional uint64 Id = 1; +} + +message TEvGetImportRequest { + optional string DatabaseName = 2; + optional TGetImportRequest Request = 1; +} + +message TGetImportResponse { + optional TImport Entry = 1; +} + +message TEvGetImportResponse { + optional TGetImportResponse Response = 1; +} + +message TCancelImportRequest { + optional uint64 Id = 1; +} + +message TEvCancelImportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 3; + optional TCancelImportRequest Request = 2; +} + +message TCancelImportResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; +} + +message TEvCancelImportResponse { + optional uint64 TxId = 1; + optional TCancelImportResponse Response = 2; +} + +message TForgetImportRequest { + optional uint64 Id = 1; +} + +message TEvForgetImportRequest { + optional uint64 TxId = 1; + optional string DatabaseName = 3; + optional TForgetImportRequest Request = 2; +} + +message TForgetImportResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; +} + +message TEvForgetImportResponse { + optional uint64 TxId = 1; + optional TForgetImportResponse Response = 2; +} + +message TListImportsRequest { + optional uint64 PageSize = 1; + optional string PageToken = 2; + optional string Kind = 3; +} + +message TEvListImportsRequest { + optional string DatabaseName = 2; + optional TListImportsRequest Request = 1; +} + +message TListImportsResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; + repeated TImport Entries = 3; + optional string NextPageToken = 4; +} + +message TEvListImportsResponse { + optional TListImportsResponse Response = 1; +} diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 36b833e88b1..849b292383a 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -90,27 +90,27 @@ message TPersQueuePartitionRequest { optional bool LastRequest = 2; // append size to last request's reservation } - message TCmdRegisterMessageGroup { - // Id of message group (SourceId) - optional bytes Id = 1; - // Range of the key to which it is allowed to write. - optional NKikimrPQ.TPartitionKeyRange PartitionKeyRange = 2; - // Starting sequence number - optional int64 StartingSeqNo = 3; - // Is the registration made after SplitMessageGroup or not - optional bool AfterSplit = 4; - } - - message TCmdDeregisterMessageGroup { - // Id of message group (SourceId) - optional bytes Id = 1; - } - - message TCmdSplitMessageGroup { - repeated TCmdDeregisterMessageGroup DeregisterGroups = 1; - repeated TCmdRegisterMessageGroup RegisterGroups = 2; - } - + message TCmdRegisterMessageGroup { + // Id of message group (SourceId) + optional bytes Id = 1; + // Range of the key to which it is allowed to write. + optional NKikimrPQ.TPartitionKeyRange PartitionKeyRange = 2; + // Starting sequence number + optional int64 StartingSeqNo = 3; + // Is the registration made after SplitMessageGroup or not + optional bool AfterSplit = 4; + } + + message TCmdDeregisterMessageGroup { + // Id of message group (SourceId) + optional bytes Id = 1; + } + + message TCmdSplitMessageGroup { + repeated TCmdDeregisterMessageGroup DeregisterGroups = 1; + repeated TCmdRegisterMessageGroup RegisterGroups = 2; + } + optional string Topic = 1; //mandatory for request for partitions optional int32 Partition = 2; //mandatory for request for partitions optional string OwnerCookie = 3; //mandatory for write @@ -126,9 +126,9 @@ message TPersQueuePartitionRequest { optional TCmdGetClientOffset CmdGetClientOffset = 10; optional TCmdGetOwnership CmdGetOwnership = 11; optional TCmdReserveBytes CmdReserveBytes = 14; - optional TCmdRegisterMessageGroup CmdRegisterMessageGroup = 20; - optional TCmdDeregisterMessageGroup CmdDeregisterMessageGroup = 21; - optional TCmdSplitMessageGroup CmdSplitMessageGroup = 22; + optional TCmdRegisterMessageGroup CmdRegisterMessageGroup = 20; + optional TCmdDeregisterMessageGroup CmdDeregisterMessageGroup = 21; + optional TCmdSplitMessageGroup CmdSplitMessageGroup = 22; optional NActorsProto.TActorId PipeClient = 15; @@ -416,8 +416,8 @@ message TPersQueuePartitionResponse { optional int64 SeqNo = 2; // will not be set if no records with such sourceId in partition optional int64 Offset = 3; //the same optional int64 WriteTimestampMS = 4; //the same - optional bool Explicit = 5; - optional NKikimrPQ.TMessageGroupInfo.EState State = 6; + optional bool Explicit = 5; + optional NKikimrPQ.TMessageGroupInfo.EState State = 6; } repeated TSourceIdInfo SourceIdInfo = 2; } diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp index 449d7d1609a..28c49a5c363 100644 --- a/ydb/core/protos/out/out.cpp +++ b/ydb/core/protos/out/out.cpp @@ -140,10 +140,10 @@ Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus, s Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::EDatashardState, stream, value) { stream << NKikimrTxDataShard::EDatashardState_Name(value); } - + Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TPDiskState::E, stream, value) { stream << NKikimrBlobStorage::TPDiskState::E_Name(value); -} +} Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TPDiskSpaceColor::E, stream, value) { stream << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(value); diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index 2362ec9cb73..7c859274496 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -126,7 +126,7 @@ message TPQConfig { optional NKikimrNodeLimits.TNodeLimitsConfig.TPersQueueNodeConfig PersQueueNodeConfig = 36; - optional bool EnableProtoSourceIdInfo = 37 [default = false]; + optional bool EnableProtoSourceIdInfo = 37 [default = false]; optional string Database = 38; @@ -210,24 +210,24 @@ message TPartitionConfig { optional TMirrorPartitionConfig MirrorFrom = 15; }; -message TPartitionKeyRange { - // Inclusive left border. Emptiness means -inf. - optional bytes FromBound = 1; - // Exclusive right border. Emptiness means +inf. - optional bytes ToBound = 2; -} - -message TMessageGroup { - // Id of message group (SourceId) - optional bytes Id = 1; - // Range of the key to which it is allowed to write. - optional TPartitionKeyRange KeyRange = 2; -} - +message TPartitionKeyRange { + // Inclusive left border. Emptiness means -inf. + optional bytes FromBound = 1; + // Exclusive right border. Emptiness means +inf. + optional bytes ToBound = 2; +} + +message TMessageGroup { + // Id of message group (SourceId) + optional bytes Id = 1; + // Range of the key to which it is allowed to write. + optional TPartitionKeyRange KeyRange = 2; +} + message TPQTabletConfig { optional uint64 CacheSize = 1 [default = 104857600]; //100Mb, per tablet optional TPartitionConfig PartitionConfig = 2; //mandatory - repeated uint32 PartitionIds = 3; //will be filled by schemeshard, don't touch. Deprecated by Partitions (id: 31) + repeated uint32 PartitionIds = 3; //will be filled by schemeshard, don't touch. Deprecated by Partitions (id: 31) optional string TopicName = 4; // also filled by schemeshard optional uint32 Version = 5; //also filled by schemeshard optional bool LocalDC = 6 [default = false]; @@ -264,44 +264,44 @@ message TPQTabletConfig { optional string YcFolderId = 27 [default = ""]; optional string YdbDatabaseId = 28 [default = ""]; optional string YdbDatabasePath = 29 [default = ""]; - - message TKeyComponentSchema { - optional string Name = 1; - optional uint32 TypeId = 2; - } - repeated TKeyComponentSchema PartitionKeySchema = 30; - - message TPartition { - optional uint32 PartitionId = 1; - optional TPartitionKeyRange KeyRange = 2; - } - repeated TPartition Partitions = 31; // filled by schemeshard -} - -message TMessageGroupInfo { - enum EState { - STATE_UNKNOWN = 0; - STATE_REGISTERED = 1; - STATE_PENDING_REGISTRATION = 2; - } - - optional uint64 SeqNo = 1; - optional uint64 Offset = 2; - optional uint64 WriteTimestamp = 3; // TInstant::TValue - optional uint64 CreateTimestamp = 4; // TInstant::TValue - optional bool Explicit = 5; - optional TPartitionKeyRange KeyRange = 6; - optional EState State = 7; -} - -message TBootstrapConfig { - repeated TMessageGroup ExplicitMessageGroups = 1; -} - + + message TKeyComponentSchema { + optional string Name = 1; + optional uint32 TypeId = 2; + } + repeated TKeyComponentSchema PartitionKeySchema = 30; + + message TPartition { + optional uint32 PartitionId = 1; + optional TPartitionKeyRange KeyRange = 2; + } + repeated TPartition Partitions = 31; // filled by schemeshard +} + +message TMessageGroupInfo { + enum EState { + STATE_UNKNOWN = 0; + STATE_REGISTERED = 1; + STATE_PENDING_REGISTRATION = 2; + } + + optional uint64 SeqNo = 1; + optional uint64 Offset = 2; + optional uint64 WriteTimestamp = 3; // TInstant::TValue + optional uint64 CreateTimestamp = 4; // TInstant::TValue + optional bool Explicit = 5; + optional TPartitionKeyRange KeyRange = 6; + optional EState State = 7; +} + +message TBootstrapConfig { + repeated TMessageGroup ExplicitMessageGroups = 1; +} + message TUpdateConfig { optional uint64 TxId = 1; optional TPQTabletConfig TabletConfig = 2; - optional TBootstrapConfig BootstrapConfig = 3; // passed only upon creation + optional TBootstrapConfig BootstrapConfig = 3; // passed only upon creation } message TUpdateBalancerConfig { //for schemeshard use only diff --git a/ydb/core/protos/replication.proto b/ydb/core/protos/replication.proto index 875ce603761..51fc9087522 100644 --- a/ydb/core/protos/replication.proto +++ b/ydb/core/protos/replication.proto @@ -1,82 +1,82 @@ -import "ydb/core/protos/base.proto"; - -package NKikimrReplication; -option java_package = "ru.yandex.kikimr.proto"; - -message TStaticCredentials { - optional string User = 1; - optional string Password = 2; -} - -message TReplicationConfig { - message TTargetEverything { - optional string DstPrefix = 1; - } - - message TTargetSpecific { - message TTarget { - optional string SrcPath = 1; - optional string DstPath = 2; - } - - repeated TTarget Targets = 1; - } - - // endpoint (host + port) and database of source cluster - optional string SrcEndpoint = 1; - optional string SrcDatabase = 2; - // credentials - oneof Credentials { - TStaticCredentials StaticCredentials = 3; - } - - // targets to be replicated - // Currently, only OLTP tables are supported. - oneof Target { - // replicate everything from source cluster - TTargetEverything Everything = 4; - // replicate specified objects - TTargetSpecific Specific = 5; - } - - optional bool InitialSync = 6; -} - -message TOperationId { - optional uint64 TxId = 1; - optional uint32 PartId = 2; -} - -message TEvCreateReplication { - optional NKikimrProto.TPathID PathId = 1; - optional TOperationId OperationId = 2; - optional TReplicationConfig Config = 3; -} - -message TEvCreateReplicationResult { - enum EStatus { - UNKNOWN = 0; - SUCCESS = 1; - ALREADY_EXISTS = 2; - } - - optional TOperationId OperationId = 1; - optional uint64 Origin = 2; - optional EStatus Status = 3; -} - -message TEvDropReplication { - optional NKikimrProto.TPathID PathId = 1; - optional TOperationId OperationId = 2; -} - -message TEvDropReplicationResult { - enum EStatus { - UNKNOWN = 0; - SUCCESS = 1; - NOT_FOUND = 2; - } - optional TOperationId OperationId = 1; - optional uint64 Origin = 2; - optional EStatus Status = 3; -} +import "ydb/core/protos/base.proto"; + +package NKikimrReplication; +option java_package = "ru.yandex.kikimr.proto"; + +message TStaticCredentials { + optional string User = 1; + optional string Password = 2; +} + +message TReplicationConfig { + message TTargetEverything { + optional string DstPrefix = 1; + } + + message TTargetSpecific { + message TTarget { + optional string SrcPath = 1; + optional string DstPath = 2; + } + + repeated TTarget Targets = 1; + } + + // endpoint (host + port) and database of source cluster + optional string SrcEndpoint = 1; + optional string SrcDatabase = 2; + // credentials + oneof Credentials { + TStaticCredentials StaticCredentials = 3; + } + + // targets to be replicated + // Currently, only OLTP tables are supported. + oneof Target { + // replicate everything from source cluster + TTargetEverything Everything = 4; + // replicate specified objects + TTargetSpecific Specific = 5; + } + + optional bool InitialSync = 6; +} + +message TOperationId { + optional uint64 TxId = 1; + optional uint32 PartId = 2; +} + +message TEvCreateReplication { + optional NKikimrProto.TPathID PathId = 1; + optional TOperationId OperationId = 2; + optional TReplicationConfig Config = 3; +} + +message TEvCreateReplicationResult { + enum EStatus { + UNKNOWN = 0; + SUCCESS = 1; + ALREADY_EXISTS = 2; + } + + optional TOperationId OperationId = 1; + optional uint64 Origin = 2; + optional EStatus Status = 3; +} + +message TEvDropReplication { + optional NKikimrProto.TPathID PathId = 1; + optional TOperationId OperationId = 2; +} + +message TEvDropReplicationResult { + enum EStatus { + UNKNOWN = 0; + SUCCESS = 1; + NOT_FOUND = 2; + } + optional TOperationId OperationId = 1; + optional uint64 Origin = 2; + optional EStatus Status = 3; +} diff --git a/ydb/core/protos/scheme_board.proto b/ydb/core/protos/scheme_board.proto index 0a28cfcd759..f063144f5d0 100644 --- a/ydb/core/protos/scheme_board.proto +++ b/ydb/core/protos/scheme_board.proto @@ -1,94 +1,94 @@ import "ydb/core/protos/flat_tx_scheme.proto"; - -package NKikimrSchemeBoard; -option java_package = "ru.yandex.kikimr.proto"; - -message TLocalPathIdRange { - optional uint64 Begin = 1; - optional uint64 End = 2; -} - -message TEvHandshake { - optional uint64 Owner = 1; - optional uint64 Generation = 2; -} - + +package NKikimrSchemeBoard; +option java_package = "ru.yandex.kikimr.proto"; + +message TLocalPathIdRange { + optional uint64 Begin = 1; + optional uint64 End = 2; +} + +message TEvHandshake { + optional uint64 Owner = 1; + optional uint64 Generation = 2; +} + // here and below // Owner is the tablet id of schemeshard witch holds the records // LocalPathId is a second part of TPathId // PathOwnerId is a first part of TPathId -message TEvUpdate { - optional uint64 Owner = 1; - optional uint64 Generation = 2; - optional TLocalPathIdRange DeletedLocalPathIds = 3; - optional string Path = 4; - optional uint64 LocalPathId = 5; - optional bool IsDeletion = 6 [default = false]; +message TEvUpdate { + optional uint64 Owner = 1; + optional uint64 Generation = 2; + optional TLocalPathIdRange DeletedLocalPathIds = 3; + optional string Path = 4; + optional uint64 LocalPathId = 5; + optional bool IsDeletion = 6 [default = false]; optional NKikimrScheme.TEvDescribeSchemeResult DescribeSchemeResult = 7; - optional bool NeedAck = 8 [default = false]; + optional bool NeedAck = 8 [default = false]; optional uint64 PathOwnerId = 9; optional TLocalPathIdRange MigratedLocalPathIds = 10; -} - -message TEvUpdateAck { - optional uint64 Owner = 1; - optional uint64 Generation = 2; - optional uint64 LocalPathId = 3; - optional uint64 Version = 4; +} + +message TEvUpdateAck { + optional uint64 Owner = 1; + optional uint64 Generation = 2; + optional uint64 LocalPathId = 3; + optional uint64 Version = 4; optional uint64 PathOwnerId = 5; -} - -message TEvCommitGeneration { - optional uint64 Owner = 1; - optional uint64 Generation = 2; -} - -message TEvSubscribe { - message TCapabilities { - optional bool AckNotifications = 1 [default = false]; - } - - optional string Path = 1; - // or +} + +message TEvCommitGeneration { + optional uint64 Owner = 1; + optional uint64 Generation = 2; +} + +message TEvSubscribe { + message TCapabilities { + optional bool AckNotifications = 1 [default = false]; + } + + optional string Path = 1; + // or optional uint64 PathOwnerId = 2; - optional uint64 LocalPathId = 3; - // common - optional uint64 DomainOwnerId = 4; - optional TCapabilities Capabilities = 5; -} - -message TEvUnsubscribe { - optional string Path = 1; - // or + optional uint64 LocalPathId = 3; + // common + optional uint64 DomainOwnerId = 4; + optional TCapabilities Capabilities = 5; +} + +message TEvUnsubscribe { + optional string Path = 1; + // or optional uint64 PathOwnerId = 2; - optional uint64 LocalPathId = 3; -} - -message TEvNotify { - optional string Path = 1; - // and/or + optional uint64 LocalPathId = 3; +} + +message TEvNotify { + optional string Path = 1; + // and/or optional uint64 PathOwnerId = 2; - optional uint64 LocalPathId = 3; - // common fields - optional bool IsDeletion = 4 [default = false]; + optional uint64 LocalPathId = 3; + // common fields + optional bool IsDeletion = 4 [default = false]; optional NKikimrScheme.TEvDescribeSchemeResult DescribeSchemeResult = 5; - optional uint64 Version = 6; - optional bool Strong = 7 [default = false]; -} - -message TEvNotifyAck { - optional uint64 Version = 1; -} - -message TEvSyncVersionRequest { - optional string Path = 1; - // and/or + optional uint64 Version = 6; + optional bool Strong = 7 [default = false]; +} + +message TEvNotifyAck { + optional uint64 Version = 1; +} + +message TEvSyncVersionRequest { + optional string Path = 1; + // and/or optional uint64 PathOwnerId = 2; - optional uint64 LocalPathId = 3; -} - -message TEvSyncVersionResponse { - optional uint64 Version = 1; - optional bool Partial = 2; -} + optional uint64 LocalPathId = 3; +} + +message TEvSyncVersionResponse { + optional uint64 Version = 1; + optional bool Partial = 2; +} diff --git a/ydb/core/protos/scheme_board_mon.proto b/ydb/core/protos/scheme_board_mon.proto index 9df3c7f6805..db3e679a841 100644 --- a/ydb/core/protos/scheme_board_mon.proto +++ b/ydb/core/protos/scheme_board_mon.proto @@ -1,169 +1,169 @@ -import "library/cpp/actors/protos/actors.proto"; - -package NKikimrSchemeBoardMon; -option java_package = "ru.yandex.kikimr.proto"; - -message TEvInfoRequest { - optional uint32 LimitRepeatedFields = 1 [default = 1000]; -} - -message TEvInfoResponse { - optional NActorsProto.TActorId Self = 1; - reserved 2; // deprecated ActivityType (enum) - optional string ActivityType = 11; - optional bool Truncated = 3 [default = false]; - oneof Response { - TReplicaResponse ReplicaResponse = 4; - TPopulatorResponse PopulatorResponse = 5; - TReplicaPopulatorResponse ReplicaPopulatorResponse = 6; - TSubscriberResponse SubscriberResponse = 7; - TSubscriberProxyResponse SubscriberProxyResponse = 8; - TReplicaSubscriberResponse ReplicaSubscriberResponse = 9; - TCacheResponse CacheResponse = 10; - } -} - -message TEvDescribeRequest { - oneof PathKind { - string Path = 1; - TPathId PathId = 2; - } -} - -message TEvDescribeResponse { - optional string Json = 1; -} - -message TPathId { - optional uint64 OwnerId = 1; - optional uint64 LocalPathId = 2; -} - -message TReplicaResponse { - message TPopulatorInfo { - optional uint64 Owner = 1; - optional uint64 Generation = 2; - optional uint64 PendingGeneration = 3; - optional NActorsProto.TActorId ActorId = 4; - } - - message TDescriptionsInfo { - optional uint64 TotalCount = 1; - optional uint64 ByPathCount = 2; - optional uint64 ByPathIdCount = 3; - } - - message TSubscriberInfo { - optional NActorsProto.TActorId ActorId = 1; - oneof PathKind { - string Path = 2; - TPathId PathId = 3; - } - } - - repeated TPopulatorInfo Populators = 1; - optional TDescriptionsInfo Descriptions = 2; - repeated TSubscriberInfo Subscribers = 3; -} - -message TPopulatorResponse { - message TUpdateAckInfo { - message TPathAckInfo { - optional TPathId PathId = 1; - optional uint64 Version = 2; - optional uint32 AcksCount = 3; - } - - optional uint64 Cookie = 1; - optional NActorsProto.TActorId AckTo = 2; - repeated TPathAckInfo PathAcks = 3; - } - - optional uint64 Owner = 1; - optional uint64 Generation = 2; - optional uint64 DescriptionsCount = 3; - optional TPathId MaxPathId = 4; - optional uint64 DelayedUpdatesCount = 5; - repeated NActorsProto.TActorId ReplicaPopulators = 6; - repeated TUpdateAckInfo UpdateAcks = 7; -} - -message TReplicaPopulatorResponse { - message TUpdateInfo { - message TVersionInfo { - optional uint64 Version = 1; - repeated uint64 TxIds = 2; - } - - optional TPathId PathId = 1; - repeated TVersionInfo Versions = 2; - } - - optional NActorsProto.TActorId Parent = 1; - optional NActorsProto.TActorId Replica = 2; - optional uint64 Owner = 3; - optional uint64 Generation = 4; - optional TPathId CurPathId = 5; - optional TPathId LastAckedPathId = 6; - optional uint32 BatchSize = 7; - optional uint32 BatchSizeLimit = 8; - repeated TUpdateInfo Updates = 9; - repeated TUpdateInfo UpdatesRequested = 10; - repeated TUpdateInfo UpdatesInFlight = 11; -} - -message TSubscriberResponse { - message TStateInfo { - optional bool Deleted = 1; - optional bool Strong = 2; - optional TPathId PathId = 3; - optional uint64 Version = 4; - optional TPathId DomainId = 5; - repeated uint64 AbandonedSchemeShards = 6; - } - - message TProxyStateInfo { - optional NActorsProto.TActorId Proxy = 1; - optional TStateInfo State = 2; - } - - optional NActorsProto.TActorId Owner = 1; - optional uint64 DomainOwnerId = 2; - repeated TProxyStateInfo ProxyStates = 3; - optional TStateInfo State = 4; - optional uint64 DelayedSyncRequest = 5; - optional uint64 CurrentSyncRequest = 6; - oneof PathKind { - string Path = 7; - TPathId PathId = 8; - } -} - -message TSubscriberProxyResponse { - optional NActorsProto.TActorId Parent = 1; - optional NActorsProto.TActorId Replica = 2; - optional NActorsProto.TActorId ReplicaSubscriber = 3; - optional uint64 DomainOwnerId = 4; - optional uint64 CurrentSyncRequest = 5; - oneof PathKind { - string Path = 6; - TPathId PathId = 7; - } -} - -message TReplicaSubscriberResponse { - optional NActorsProto.TActorId Parent = 1; - optional NActorsProto.TActorId Replica = 2; - optional uint64 DomainOwnerId = 3; - optional uint64 CurrentSyncRequest = 4; - oneof PathKind { - string Path = 5; - TPathId PathId = 6; - } -} - -message TCacheResponse { - optional uint64 ItemsTotalCount = 1; - optional uint64 ItemsByPathCount = 2; - optional uint64 ItemsByPathIdCount = 3; -} +import "library/cpp/actors/protos/actors.proto"; + +package NKikimrSchemeBoardMon; +option java_package = "ru.yandex.kikimr.proto"; + +message TEvInfoRequest { + optional uint32 LimitRepeatedFields = 1 [default = 1000]; +} + +message TEvInfoResponse { + optional NActorsProto.TActorId Self = 1; + reserved 2; // deprecated ActivityType (enum) + optional string ActivityType = 11; + optional bool Truncated = 3 [default = false]; + oneof Response { + TReplicaResponse ReplicaResponse = 4; + TPopulatorResponse PopulatorResponse = 5; + TReplicaPopulatorResponse ReplicaPopulatorResponse = 6; + TSubscriberResponse SubscriberResponse = 7; + TSubscriberProxyResponse SubscriberProxyResponse = 8; + TReplicaSubscriberResponse ReplicaSubscriberResponse = 9; + TCacheResponse CacheResponse = 10; + } +} + +message TEvDescribeRequest { + oneof PathKind { + string Path = 1; + TPathId PathId = 2; + } +} + +message TEvDescribeResponse { + optional string Json = 1; +} + +message TPathId { + optional uint64 OwnerId = 1; + optional uint64 LocalPathId = 2; +} + +message TReplicaResponse { + message TPopulatorInfo { + optional uint64 Owner = 1; + optional uint64 Generation = 2; + optional uint64 PendingGeneration = 3; + optional NActorsProto.TActorId ActorId = 4; + } + + message TDescriptionsInfo { + optional uint64 TotalCount = 1; + optional uint64 ByPathCount = 2; + optional uint64 ByPathIdCount = 3; + } + + message TSubscriberInfo { + optional NActorsProto.TActorId ActorId = 1; + oneof PathKind { + string Path = 2; + TPathId PathId = 3; + } + } + + repeated TPopulatorInfo Populators = 1; + optional TDescriptionsInfo Descriptions = 2; + repeated TSubscriberInfo Subscribers = 3; +} + +message TPopulatorResponse { + message TUpdateAckInfo { + message TPathAckInfo { + optional TPathId PathId = 1; + optional uint64 Version = 2; + optional uint32 AcksCount = 3; + } + + optional uint64 Cookie = 1; + optional NActorsProto.TActorId AckTo = 2; + repeated TPathAckInfo PathAcks = 3; + } + + optional uint64 Owner = 1; + optional uint64 Generation = 2; + optional uint64 DescriptionsCount = 3; + optional TPathId MaxPathId = 4; + optional uint64 DelayedUpdatesCount = 5; + repeated NActorsProto.TActorId ReplicaPopulators = 6; + repeated TUpdateAckInfo UpdateAcks = 7; +} + +message TReplicaPopulatorResponse { + message TUpdateInfo { + message TVersionInfo { + optional uint64 Version = 1; + repeated uint64 TxIds = 2; + } + + optional TPathId PathId = 1; + repeated TVersionInfo Versions = 2; + } + + optional NActorsProto.TActorId Parent = 1; + optional NActorsProto.TActorId Replica = 2; + optional uint64 Owner = 3; + optional uint64 Generation = 4; + optional TPathId CurPathId = 5; + optional TPathId LastAckedPathId = 6; + optional uint32 BatchSize = 7; + optional uint32 BatchSizeLimit = 8; + repeated TUpdateInfo Updates = 9; + repeated TUpdateInfo UpdatesRequested = 10; + repeated TUpdateInfo UpdatesInFlight = 11; +} + +message TSubscriberResponse { + message TStateInfo { + optional bool Deleted = 1; + optional bool Strong = 2; + optional TPathId PathId = 3; + optional uint64 Version = 4; + optional TPathId DomainId = 5; + repeated uint64 AbandonedSchemeShards = 6; + } + + message TProxyStateInfo { + optional NActorsProto.TActorId Proxy = 1; + optional TStateInfo State = 2; + } + + optional NActorsProto.TActorId Owner = 1; + optional uint64 DomainOwnerId = 2; + repeated TProxyStateInfo ProxyStates = 3; + optional TStateInfo State = 4; + optional uint64 DelayedSyncRequest = 5; + optional uint64 CurrentSyncRequest = 6; + oneof PathKind { + string Path = 7; + TPathId PathId = 8; + } +} + +message TSubscriberProxyResponse { + optional NActorsProto.TActorId Parent = 1; + optional NActorsProto.TActorId Replica = 2; + optional NActorsProto.TActorId ReplicaSubscriber = 3; + optional uint64 DomainOwnerId = 4; + optional uint64 CurrentSyncRequest = 5; + oneof PathKind { + string Path = 6; + TPathId PathId = 7; + } +} + +message TReplicaSubscriberResponse { + optional NActorsProto.TActorId Parent = 1; + optional NActorsProto.TActorId Replica = 2; + optional uint64 DomainOwnerId = 3; + optional uint64 CurrentSyncRequest = 4; + oneof PathKind { + string Path = 5; + TPathId PathId = 6; + } +} + +message TCacheResponse { + optional uint64 ItemsTotalCount = 1; + optional uint64 ItemsByPathCount = 2; + optional uint64 ItemsByPathIdCount = 3; +} diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 24ba6e08edf..c17c8a7dc37 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -143,7 +143,7 @@ enum EServiceKikimr { TX_PROXY_SCHEME_CACHE = 401; MSGBUS_PROXY = 403; TX_ALLOCATOR = 402; - TX_ALLOCATOR_CLIENT = 404; + TX_ALLOCATOR_CLIENT = 404; LONG_TX_SERVICE = 405; LOGGER = 410; @@ -231,11 +231,11 @@ enum EServiceKikimr { CONFIGS_DISPATCHER = 560; CONFIGS_CACHE = 561; - // SchemeBoard section - SCHEME_BOARD_REPLICA = 570; - SCHEME_BOARD_SUBSCRIBER = 571; - SCHEME_BOARD_POPULATOR = 572; - + // SchemeBoard section + SCHEME_BOARD_REPLICA = 570; + SCHEME_BOARD_SUBSCRIBER = 571; + SCHEME_BOARD_POPULATOR = 572; + // Distributed Storage DS_PROXY_NODE_MON = 600; @@ -255,7 +255,7 @@ enum EServiceKikimr { YF_CALL_BALANCER = 708; YF_DESCRIPTIONS_SERVICE = 709; YF_WORKER = 710; - + NET_CLASSIFIER = 711; PERSQUEUE_CLUSTER_TRACKER = 712; @@ -266,11 +266,11 @@ enum EServiceKikimr { YQL_NODES_MANAGER = 715; - // Exports (& imports) - EXPORT = 800; - DATASHARD_RESTORE = 801; - IMPORT = 802; - S3_WRAPPER = 803; + // Exports (& imports) + EXPORT = 800; + DATASHARD_RESTORE = 801; + IMPORT = 802; + S3_WRAPPER = 803; // System views SYSTEM_VIEWS = 900; @@ -290,19 +290,19 @@ enum EServiceKikimr { STREAMS_GRAND_LEADER_SERVICE = 1018; STREAMS_META_STORAGE_SERVICE = 1019; STREAMS_GRAPH_LEADER = 1020; - + // YandexQuery YQ_CONTROL_PLANE_STORAGE = 1021; YQ_CONTROL_PLANE_PROXY= 1022; YQ_TEST_CONNECTION = 1023; - // Change exchange (async indexes & CDC) - CHANGE_EXCHANGE = 1100; + // Change exchange (async indexes & CDC) + CHANGE_EXCHANGE = 1100; YDB_SDK = 1101; // Log component for logger in YDB SDK driver. - - // Replication - REPLICATION_CONTROLLER = 1200; + + // Replication + REPLICATION_CONTROLLER = 1200; }; message TActivity { @@ -694,19 +694,19 @@ message TActivity { BS_MIGRATION_ACTOR = 379; SQS_GARBAGE_COLLECTOR_ACTOR = 380; SYSTEM_VIEW_SERVICE = 381; - CMS_SENTINEL_ACTOR = 382; - CMS_SENTINEL_CONFIG_UPDATER_ACTOR = 383; - CMS_SENTINEL_STATE_UPDATER_ACTOR = 384; - CMS_SENTINEL_STATUS_CHANGER_ACTOR = 385; + CMS_SENTINEL_ACTOR = 382; + CMS_SENTINEL_CONFIG_UPDATER_ACTOR = 383; + CMS_SENTINEL_STATE_UPDATER_ACTOR = 384; + CMS_SENTINEL_STATUS_CHANGER_ACTOR = 385; TENANT_NODES_ENUMERATION = 386; SCHEME_BOARD_SYNCHRONIZER_ACTOR = 387; SCHEME_BOARD_REPLICA_SYNCHRONIZER_ACTOR = 388; SYSTEM_VIEW_PART_STATS_COLLECTOR = 389; BLOCKSTORE_DISK_AGENT = 390; BLOCKSTORE_DISK_REGISTRY = 391; - EXPORT_SCAN_ACTOR = 392; - EXPORT_YT_UPLOADER_ACTOR = 393; - EXPORT_S3_UPLOADER_ACTOR = 394; + EXPORT_SCAN_ACTOR = 392; + EXPORT_YT_UPLOADER_ACTOR = 393; + EXPORT_S3_UPLOADER_ACTOR = 394; YF_SYNC_SERVICE_ACTOR = 395; YF_SYNC_SERVICE_WORKER_ACTOR = 396; YF_DESCRIPTIONS_SERVICE_ACTOR = 397; @@ -755,7 +755,7 @@ message TActivity { YF_REQUEST_CALL_FUNCTION_ACTOR = 440; SCHEME_BOARD_DB_RESOLVER = 441; YF_LOGGER_ACTOR = 442; - CONDITIONAL_ERASE_ROWS_SCAN_ACTOR = 443; + CONDITIONAL_ERASE_ROWS_SCAN_ACTOR = 443; BLOCKSTORE_VOLUME_BALANCER = 444; BLOCKSTORE_DISK_REGISTRY_WORKER = 445; BLOCKSTORE_DISK_AGENT_WORKER = 446; @@ -826,9 +826,9 @@ message TActivity { MONITORING_SERVICE = 521; MONITORING_REQUEST = 522; ACTOR_SERVICE_CACHE = 523; - CHANGE_SENDER_ACTOR = 524; - CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN = 525; - CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION = 526; + CHANGE_SENDER_ACTOR = 524; + CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN = 525; + CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION = 526; YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527; YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528; YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529; @@ -852,20 +852,20 @@ message TActivity { INTERCONNECT_PROXY_WRAPPER = 546; CLOUD_STORAGE_HIVE_PROXY = 547; YQL_PRIVATE_NODES_HEALTH_CHECK_ACTOR = 548; - CHANGE_SENDER_CDC_ACTOR_MAIN = 549; + CHANGE_SENDER_CDC_ACTOR_MAIN = 549; KQP_TEST_WORKLOAD = 550; - PQ_PARTITION_WRITER_ACTOR = 551; - CHANGE_SENDER_CDC_ACTOR_PARTITION = 552; + PQ_PARTITION_WRITER_ACTOR = 551; + CHANGE_SENDER_CDC_ACTOR_PARTITION = 552; KQP_SESSION_ACTOR = 553; - CHANGE_EXCHANGE_SPLIT_ACTOR = 554; + CHANGE_EXCHANGE_SPLIT_ACTOR = 554; BS_DEFRAG_REWRITER = 555; BS_DEFRAG_SCHEDULER = 556; BS_DEFRAG_PLANNER = 557; - REPLICATION_CONTROLLER_ACTOR = 558; - REPLICATION_CONTROLLER_DISCOVERER = 559; - REPLICATION_CONTROLLER_STREAM_CREATOR = 561; - YQ_TEST_CONNECTION_ACTOR = 560; - REPLICATION_CONTROLLER_DST_CREATOR = 562; + REPLICATION_CONTROLLER_ACTOR = 558; + REPLICATION_CONTROLLER_DISCOVERER = 559; + REPLICATION_CONTROLLER_STREAM_CREATOR = 561; + YQ_TEST_CONNECTION_ACTOR = 560; + REPLICATION_CONTROLLER_DST_CREATOR = 562; BLOCKSTORE_STATS_SERVICE = 563; }; }; diff --git a/ydb/core/protos/subdomains.proto b/ydb/core/protos/subdomains.proto index 4c1c25cae8f..5d665ff3813 100644 --- a/ydb/core/protos/subdomains.proto +++ b/ydb/core/protos/subdomains.proto @@ -4,11 +4,11 @@ import "ydb/library/login/protos/login.proto"; package NKikimrSubDomains; -message TDomainKey { - optional fixed64 SchemeShard = 1; - optional fixed64 PathId = 2; -} - +message TDomainKey { + optional fixed64 SchemeShard = 1; + optional fixed64 PathId = 2; +} + message TSubDomainSettings { optional uint64 PlanResolution = 1; optional uint64 Coordinators = 2; @@ -18,7 +18,7 @@ message TSubDomainSettings { repeated NKikimrStoragePool.TStoragePool StoragePools = 6; optional bool ExternalSchemeShard = 7 [default = false]; optional bool ExternalHive = 8 [default = false]; - optional TDomainKey ResourcesDomainKey = 9; + optional TDomainKey ResourcesDomainKey = 9; optional bool ExternalSysViewProcessor = 10 [default = false]; optional TSchemeQuotas DeclaredSchemeQuotas = 11; optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 12; @@ -38,17 +38,17 @@ message TProcessingParams { //put there SubSchemeShard and SubHive at the future } -message TDiskSpaceUsage { - message TTables { - // in bytes - optional uint64 TotalSize = 1; - optional uint64 DataSize = 2; - optional uint64 IndexSize = 3; - } - - optional TTables Tables = 1; -} - +message TDiskSpaceUsage { + message TTables { + // in bytes + optional uint64 TotalSize = 1; + optional uint64 DataSize = 2; + optional uint64 IndexSize = 3; + } + + optional TTables Tables = 1; +} + message TDomainState { optional bool DiskQuotaExceeded = 1; } @@ -65,10 +65,10 @@ message TDomainDescription { optional uint64 PathsLimit = 7; optional uint64 ShardsInside = 8; optional uint64 ShardsLimit = 9; - - optional TDomainKey ResourcesDomainKey = 10; - - optional TDiskSpaceUsage DiskSpaceUsage = 11; // TODO: temp for SLYDB-95 + + optional TDomainKey ResourcesDomainKey = 10; + + optional TDiskSpaceUsage DiskSpaceUsage = 11; // TODO: temp for SLYDB-95 optional TSchemeQuotas DeclaredSchemeQuotas = 12; diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto index 51953def8d6..cfc03662725 100644 --- a/ydb/core/protos/sys_view.proto +++ b/ydb/core/protos/sys_view.proto @@ -10,12 +10,12 @@ message TPartitionStatsKey { optional uint64 PartIdx = 3; } -message TTtlStats { - optional uint64 LastRunTime = 1; // milliseconds since epoch - optional uint64 LastRowsProcessed = 2; - optional uint64 LastRowsErased = 3; -} - +message TTtlStats { + optional uint64 LastRunTime = 1; // milliseconds since epoch + optional uint64 LastRowsProcessed = 2; + optional uint64 LastRowsErased = 3; +} + message TPartitionStats { optional uint64 DataSize = 1; optional uint64 RowCount = 2; @@ -41,8 +41,8 @@ message TPartitionStats { optional uint64 PlannedTxCompleted = 17; optional uint64 TxRejectedByOverload = 18; optional uint64 TxRejectedBySpace = 19; - - optional TTtlStats TtlStats = 20; + + optional TTtlStats TtlStats = 20; } message TPartitionStatsResult { diff --git a/ydb/core/protos/tablet.proto b/ydb/core/protos/tablet.proto index f90bc745ae3..f0bec238e78 100644 --- a/ydb/core/protos/tablet.proto +++ b/ydb/core/protos/tablet.proto @@ -45,7 +45,7 @@ message TTabletTypes { ColumnShard = 35; TestShard = 36; SequenceShard = 37; - ReplicationController = 38; + ReplicationController = 38; // when adding a new tablet type and keeping parse compatibility with the old version // rename existing reserved item to desired one, and add new reserved item to @@ -53,7 +53,7 @@ message TTabletTypes { Reserved39 = 39; Reserved40 = 40; Reserved41 = 41; - Reserved42 = 42; + Reserved42 = 42; UserTypeStart = 255; TypeInvalid = -1; diff --git a/ydb/core/protos/tenant_pool.proto b/ydb/core/protos/tenant_pool.proto index 43974a74e48..b4d39fbab71 100644 --- a/ydb/core/protos/tenant_pool.proto +++ b/ydb/core/protos/tenant_pool.proto @@ -24,13 +24,13 @@ message TTenantPoolConfig { optional string NodeType = 3; } -enum EState { - STATE_UNKNOWN = 0; - TENANT_ASSIGNED = 1; - TENANT_OK = 2; - TENANT_UNKNOWN = 3; -} - +enum EState { + STATE_UNKNOWN = 0; + TENANT_ASSIGNED = 1; + TENANT_OK = 2; + TENANT_UNKNOWN = 3; +} + message TSlotStatus { optional string Id = 1; optional string Type = 2; @@ -38,8 +38,8 @@ message TSlotStatus { optional NKikimrTabletBase.TMetrics ResourceLimit = 4; optional string Label = 5; repeated NKikimrSchemeOp.TUserAttribute TenantAttributes = 6; - optional EState State = 7; - optional NKikimrSubDomains.TDomainKey DomainKey = 8; + optional EState State = 7; + optional NKikimrSubDomains.TDomainKey DomainKey = 8; } message TGetStatus { @@ -73,8 +73,8 @@ message TConfigureSlotResult { } message TTakeOwnership { - optional uint64 Generation = 1; - optional uint64 SeqNo = 2; + optional uint64 Generation = 1; + optional uint64 SeqNo = 2; } message TLostOwnership { diff --git a/ydb/core/protos/tenant_slot_broker.proto b/ydb/core/protos/tenant_slot_broker.proto index ef154371ac5..bccc7f7dc7b 100644 --- a/ydb/core/protos/tenant_slot_broker.proto +++ b/ydb/core/protos/tenant_slot_broker.proto @@ -4,7 +4,7 @@ package NKikimrTenantSlotBroker; option java_package = "ru.yandex.kikimr.proto"; message TConfig { - optional uint64 PendingSlotTimeout = 1 [default = 15000000]; // 15 seconds by default + optional uint64 PendingSlotTimeout = 1 [default = 15000000]; // 15 seconds by default } message TSlotAllocation { @@ -55,7 +55,7 @@ message TAlterTenant { message TRegisterPool { optional NActorsProto.TActorId ClientId = 1; - optional uint64 SeqNo = 2; + optional uint64 SeqNo = 2; } message TListTenants { diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index f7dcc0aca31..89863b25400 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -55,8 +55,8 @@ message TEvGetShardStateResult { message TShardOpResult { optional bool Success = 1; optional string Explain = 2; - optional uint64 BytesProcessed = 3; - optional uint64 RowsProcessed = 4; + optional uint64 BytesProcessed = 3; + optional uint64 RowsProcessed = 4; } message TEvSchemaChanged { @@ -325,15 +325,15 @@ message TDropPersistentSnapshot { } message TInitiateBuildIndex { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional string SnapshotName = 2; - optional uint64 TableSchemaVersion = 3; + optional uint64 TableSchemaVersion = 3; optional NKikimrSchemeOp.TIndexDescription IndexDescription = 4; } message TFinalizeBuildIndex { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional uint64 SnapshotStep = 2; optional uint64 SnapshotTxId = 3; @@ -344,45 +344,45 @@ message TFinalizeBuildIndex { } message TDropIndexNotice { - // Path id of the table that the index is being dropped from - optional NKikimrProto.TPathID PathId = 1; - optional uint64 TableSchemaVersion = 2; + // Path id of the table that the index is being dropped from + optional NKikimrProto.TPathID PathId = 1; + optional uint64 TableSchemaVersion = 2; - // Path id of the index being dropped - optional NKikimrProto.TPathID IndexPathId = 3; + // Path id of the index being dropped + optional NKikimrProto.TPathID IndexPathId = 3; } -message TCreateCdcStreamNotice { - optional NKikimrProto.TPathID PathId = 1; - optional uint64 TableSchemaVersion = 2; +message TCreateCdcStreamNotice { + optional NKikimrProto.TPathID PathId = 1; + optional uint64 TableSchemaVersion = 2; optional NKikimrSchemeOp.TCdcStreamDescription StreamDescription = 3; -} - -message TAlterCdcStreamNotice { - optional NKikimrProto.TPathID PathId = 1; - optional uint64 TableSchemaVersion = 2; +} + +message TAlterCdcStreamNotice { + optional NKikimrProto.TPathID PathId = 1; + optional uint64 TableSchemaVersion = 2; optional NKikimrSchemeOp.TCdcStreamDescription StreamDescription = 3; -} - -message TDropCdcStreamNotice { - optional NKikimrProto.TPathID PathId = 1; - optional uint64 TableSchemaVersion = 2; - optional NKikimrProto.TPathID StreamPathId = 3; -} - -message TAsyncIndexInfo { -} - +} + +message TDropCdcStreamNotice { + optional NKikimrProto.TPathID PathId = 1; + optional uint64 TableSchemaVersion = 2; + optional NKikimrProto.TPathID StreamPathId = 3; +} + +message TAsyncIndexInfo { +} + message TMoveTable { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional uint64 TableSchemaVersion = 2; - optional NKikimrProto.TPathID DstPathId = 3; + optional NKikimrProto.TPathID DstPathId = 3; optional string DstPath = 4; message TRemapIndexPathId { - optional NKikimrProto.TPathID PathId = 1; - optional NKikimrProto.TPathID DstPathId = 2; + optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID DstPathId = 2; } repeated TRemapIndexPathId ReMapIndexes = 5; @@ -406,41 +406,41 @@ message TFlatSchemeTransaction { optional TInitiateBuildIndex InitiateBuildIndex = 12; optional TFinalizeBuildIndex FinalizeBuildIndex = 13; optional TDropIndexNotice DropIndexNotice = 14; - + optional NKikimrSchemeOp.TRestoreTask Restore = 15; - - optional TAsyncIndexInfo AsyncIndexInfo = 16; + + optional TAsyncIndexInfo AsyncIndexInfo = 16; optional TMoveTable MoveTable = 17; - - optional TCreateCdcStreamNotice CreateCdcStreamNotice = 18; - optional TAlterCdcStreamNotice AlterCdcStreamNotice = 19; - optional TDropCdcStreamNotice DropCdcStreamNotice = 20; -} - -message TDistributedEraseTransaction { - message TDependent { - optional uint64 ShardId = 1; - } - - message TDependency { - optional uint64 ShardId = 1; - optional bytes PresentRows = 2; // bitmap - } - - optional TEvEraseRowsRequest EraseRowsRequest = 1; - repeated TDependent Dependents = 2; - repeated TDependency Dependencies = 3; - - // Identifiers (in term of main table) & values of indexed columns - repeated uint32 IndexColumnIds = 4; - repeated bytes IndexColumns = 5; // SerilializedCellVector -} - -message TDistributedEraseRS { - optional bytes ConfirmedRows = 1; // bitmap -} - + + optional TCreateCdcStreamNotice CreateCdcStreamNotice = 18; + optional TAlterCdcStreamNotice AlterCdcStreamNotice = 19; + optional TDropCdcStreamNotice DropCdcStreamNotice = 20; +} + +message TDistributedEraseTransaction { + message TDependent { + optional uint64 ShardId = 1; + } + + message TDependency { + optional uint64 ShardId = 1; + optional bytes PresentRows = 2; // bitmap + } + + optional TEvEraseRowsRequest EraseRowsRequest = 1; + repeated TDependent Dependents = 2; + repeated TDependency Dependencies = 3; + + // Identifiers (in term of main table) & values of indexed columns + repeated uint32 IndexColumnIds = 4; + repeated bytes IndexColumns = 5; // SerilializedCellVector +} + +message TDistributedEraseRS { + optional bytes ConfirmedRows = 1; // bitmap +} + // This transaction commits all changes previously written using some WriteTxId message TCommitWritesTransaction { // TODO: add lock checking in the future @@ -457,7 +457,7 @@ enum ETransactionKind { DEPRECATED_4 = 4; //TX_KIND_SNAPSHOT_POLICY TX_KIND_SCAN = 5; TX_KIND_SNAPSHOT = 6; - TX_KIND_DISTRIBUTED_ERASE = 7; + TX_KIND_DISTRIBUTED_ERASE = 7; TX_KIND_COMMIT_WRITES = 8; // Values [100, inf) are used for internal datashard operation @@ -682,7 +682,7 @@ message TEvSplitTransferSnapshot { optional uint64 MvccIncompleteEdgeTxId = 11; optional uint64 MvccLowWatermarkStep = 12; optional uint64 MvccLowWatermarkTxId = 13; - optional bool WaitForActivation = 14; + optional bool WaitForActivation = 14; // Generation of the sender tablet optional uint64 SrcTabletGeneration = 15; @@ -712,11 +712,11 @@ message TEvCancelBackup { optional uint64 TableId = 2; } -message TEvCancelRestore { - optional uint64 RestoreTxId = 1; - optional uint64 TableId = 2; -} - +message TEvCancelRestore { + optional uint64 RestoreTxId = 1; + optional uint64 TableId = 2; +} + message TEvGetTableStats { optional uint64 TableId = 1; optional uint64 DataSizeResolution = 2; @@ -1218,102 +1218,102 @@ message TEvMigrateSchemeShardResponse { optional fixed64 TabletId = 2; } -message TExpirationCondition { - // erase if valueof(ColumnId) < WallClockTimestamp - optional uint32 ColumnId = 1; // Must contain date type - optional uint64 WallClockTimestamp = 2; +message TExpirationCondition { + // erase if valueof(ColumnId) < WallClockTimestamp + optional uint32 ColumnId = 1; // Must contain date type + optional uint64 WallClockTimestamp = 2; optional NKikimrSchemeOp.TTTLSettings.EUnit ColumnUnit = 3; -} - -message TEvEraseRowsRequest { - optional uint64 TableId = 1; - repeated uint32 KeyColumnIds = 2; - repeated bytes KeyColumns = 3; // SerilializedCellVector - optional uint64 SchemaVersion = 5; - - oneof Condition { - TExpirationCondition Expiration = 4; - } -} - -message TEvEraseRowsResponse { - enum EStatus { - OK = 0; - BAD_REQUEST = 1; - SCHEME_ERROR = 2; - EXEC_ERROR = 3; - WRONG_SHARD_STATE = 4; - UNKNOWN = 5; - // coordinator - COORDINATOR_DECLINED = 10; - COORDINATOR_UNKNOWN = 11; - // shard - SHARD_NOT_AVAILABLE = 20; - SHARD_ABORTED = 21; - SHARD_CANCELLED = 22; - SHARD_RESULT_UNAVAILABLE = 23; - SHARD_OVERLOADED = 24; - SHARD_TRY_LATER = 25; - SHARD_EXEC_ERROR = 26; - SHARD_UNKNOWN = 27; - } - - optional uint64 TabletID = 1; - optional EStatus Status = 2; - optional string ErrorDescription = 3; -} - -message TEvConditionalEraseRowsRequest { - message TIndexDescription { - message TKeyMap { - optional uint32 IndexColumnId = 1; - optional uint32 MainColumnId = 2; - } - - optional uint64 OwnerId = 1; - optional uint64 PathId = 2; - optional uint64 SchemaVersion = 3; - repeated TKeyMap KeyMap = 4; // sorted - }; - - message TLimits { - optional uint32 BatchMaxBytes = 1 [default = 512000]; - optional uint32 BatchMinKeys = 2 [default = 1000]; - optional uint32 BatchMaxKeys = 3 [default = 50000]; - }; - - optional uint64 TableId = 1; - optional uint64 SchemaVersion = 3; - repeated TIndexDescription Indexes = 4; - optional TLimits Limits = 5; - - oneof Condition { - TExpirationCondition Expiration = 2; - } -} - -message TEvConditionalEraseRowsResponse { - enum EStatus { - ACCEPTED = 0; - PARTIAL = 1; - OK = 2; - BAD_REQUEST = 3; - ABORTED = 4; - ERASE_ERROR = 5; // error during underlying erase execution - OVERLOADED = 6; // another conditional erase in flight - SCHEME_ERROR = 7; - } - - message TStats { - optional uint64 RowsProcessed = 1; - optional uint64 RowsErased = 2; - } - - optional uint64 TabletID = 1; - optional EStatus Status = 2; - optional string ErrorDescription = 3; - optional TStats Stats = 4; -} +} + +message TEvEraseRowsRequest { + optional uint64 TableId = 1; + repeated uint32 KeyColumnIds = 2; + repeated bytes KeyColumns = 3; // SerilializedCellVector + optional uint64 SchemaVersion = 5; + + oneof Condition { + TExpirationCondition Expiration = 4; + } +} + +message TEvEraseRowsResponse { + enum EStatus { + OK = 0; + BAD_REQUEST = 1; + SCHEME_ERROR = 2; + EXEC_ERROR = 3; + WRONG_SHARD_STATE = 4; + UNKNOWN = 5; + // coordinator + COORDINATOR_DECLINED = 10; + COORDINATOR_UNKNOWN = 11; + // shard + SHARD_NOT_AVAILABLE = 20; + SHARD_ABORTED = 21; + SHARD_CANCELLED = 22; + SHARD_RESULT_UNAVAILABLE = 23; + SHARD_OVERLOADED = 24; + SHARD_TRY_LATER = 25; + SHARD_EXEC_ERROR = 26; + SHARD_UNKNOWN = 27; + } + + optional uint64 TabletID = 1; + optional EStatus Status = 2; + optional string ErrorDescription = 3; +} + +message TEvConditionalEraseRowsRequest { + message TIndexDescription { + message TKeyMap { + optional uint32 IndexColumnId = 1; + optional uint32 MainColumnId = 2; + } + + optional uint64 OwnerId = 1; + optional uint64 PathId = 2; + optional uint64 SchemaVersion = 3; + repeated TKeyMap KeyMap = 4; // sorted + }; + + message TLimits { + optional uint32 BatchMaxBytes = 1 [default = 512000]; + optional uint32 BatchMinKeys = 2 [default = 1000]; + optional uint32 BatchMaxKeys = 3 [default = 50000]; + }; + + optional uint64 TableId = 1; + optional uint64 SchemaVersion = 3; + repeated TIndexDescription Indexes = 4; + optional TLimits Limits = 5; + + oneof Condition { + TExpirationCondition Expiration = 2; + } +} + +message TEvConditionalEraseRowsResponse { + enum EStatus { + ACCEPTED = 0; + PARTIAL = 1; + OK = 2; + BAD_REQUEST = 3; + ABORTED = 4; + ERASE_ERROR = 5; // error during underlying erase execution + OVERLOADED = 6; // another conditional erase in flight + SCHEME_ERROR = 7; + } + + message TStats { + optional uint64 RowsProcessed = 1; + optional uint64 RowsErased = 2; + } + + optional uint64 TabletID = 1; + optional EStatus Status = 2; + optional string ErrorDescription = 3; + optional TStats Stats = 4; +} message TEvBuildIndexCreateRequest { optional uint64 BuildIndexId = 1; // unique id of build index proccess @@ -1399,7 +1399,7 @@ message TEvKqpScan { } message TEvCompactTable { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; optional bool CompactBorrowed = 2; } @@ -1411,16 +1411,16 @@ message TEvCompactTableResult { }; optional uint64 TabletId = 1; - optional NKikimrProto.TPathID PathId = 2; + optional NKikimrProto.TPathID PathId = 2; optional EStatus Status = 3; } message TEvCompactBorrowed { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; } message TEvGetCompactTableStats { - optional NKikimrProto.TPathID PathId = 1; + optional NKikimrProto.TPathID PathId = 1; } message TEvGetCompactTableStatsResult { diff --git a/ydb/core/protos/tx_proxy.proto b/ydb/core/protos/tx_proxy.proto index d3572272aa1..dedd2f9cbb5 100644 --- a/ydb/core/protos/tx_proxy.proto +++ b/ydb/core/protos/tx_proxy.proto @@ -192,8 +192,8 @@ message TEvProposeTransaction { optional string UserToken = 4; // already built and serialized user's token optional bool StreamResponse = 5; optional uint64 CancelAfterMs = 6; - optional string DatabaseName = 7; - optional string RequestType = 8; + optional string DatabaseName = 7; + optional string RequestType = 8; } message TTxProxyTimings { @@ -254,7 +254,7 @@ message TEvNavigate { optional NKikimrSchemeOp.TDescribePath DescribePath = 5; optional string UserToken = 6; // already built and serialized user's token - optional string DatabaseName = 7; + optional string DatabaseName = 7; } // This message is no longer used @@ -275,26 +275,26 @@ message TEvInvalidateTable { optional uint64 TableId = 2; } -message TEvExportRequest { - optional string DatabaseName = 1; - optional string UserToken = 2; - oneof Request { - NKikimrExport.TCreateExportRequest CreateExport = 3; - NKikimrExport.TGetExportRequest GetExport = 4; - NKikimrExport.TCancelExportRequest CancelExport = 5; - NKikimrExport.TForgetExportRequest ForgetExport = 6; - NKikimrExport.TListExportsRequest ListExports = 7; - } +message TEvExportRequest { + optional string DatabaseName = 1; + optional string UserToken = 2; + oneof Request { + NKikimrExport.TCreateExportRequest CreateExport = 3; + NKikimrExport.TGetExportRequest GetExport = 4; + NKikimrExport.TCancelExportRequest CancelExport = 5; + NKikimrExport.TForgetExportRequest ForgetExport = 6; + NKikimrExport.TListExportsRequest ListExports = 7; + } } -message TEvExportResponse { +message TEvExportResponse { optional uint32 Status = 1; - optional string ErrorDescription = 7; - oneof Response { - NKikimrExport.TCreateExportResponse CreateExport = 2; - NKikimrExport.TGetExportResponse GetExport = 3; - NKikimrExport.TCancelExportResponse CancelExport = 4; - NKikimrExport.TForgetExportResponse ForgetExport = 5; - NKikimrExport.TListExportsResponse ListExports = 6; - } + optional string ErrorDescription = 7; + oneof Response { + NKikimrExport.TCreateExportResponse CreateExport = 2; + NKikimrExport.TGetExportResponse GetExport = 3; + NKikimrExport.TCancelExportResponse CancelExport = 4; + NKikimrExport.TForgetExportResponse ForgetExport = 5; + NKikimrExport.TListExportsResponse ListExports = 6; + } } diff --git a/ydb/core/protos/tx_sequenceshard.proto b/ydb/core/protos/tx_sequenceshard.proto index d35f7d89f06..dd3d203250a 100644 --- a/ydb/core/protos/tx_sequenceshard.proto +++ b/ydb/core/protos/tx_sequenceshard.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -import "ydb/core/protos/base.proto"; +import "ydb/core/protos/base.proto"; package NKikimrTxSequenceShard; option java_package = "ru.yandex.kikimr.proto"; @@ -18,7 +18,7 @@ message TEvMarkSchemeShardPipe { } message TEvCreateSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; oneof OptionalMinValue { @@ -56,7 +56,7 @@ message TEvCreateSequenceResult { } message TEvAllocateSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 Cache = 2; } @@ -80,7 +80,7 @@ message TEvAllocateSequenceResult { } message TEvDropSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; } @@ -100,7 +100,7 @@ message TEvDropSequenceResult { } message TEvUpdateSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; oneof ChangeMinValue { @@ -147,7 +147,7 @@ message TEvUpdateSequenceResult { } message TEvFreezeSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; } @@ -177,7 +177,7 @@ message TEvFreezeSequenceResult { } message TEvRestoreSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; sint64 MinValue = 4; @@ -205,7 +205,7 @@ message TEvRestoreSequenceResult { } message TEvRedirectSequence { - NKikimrProto.TPathID PathId = 1; + NKikimrProto.TPathID PathId = 1; uint64 TxId = 2; uint64 TxPartId = 3; uint64 RedirectTo = 4; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 4316b624171..70bb65514cd 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -27,7 +27,7 @@ SRCS( blockstore_config.proto filestore_config.proto bootstrapper.proto - change_exchange.proto + change_exchange.proto channel_purpose.proto cms.proto config.proto @@ -46,7 +46,7 @@ SRCS( counters_keyvalue.proto counters_olapshard.proto counters_pq.proto - counters_replication.proto + counters_replication.proto counters_schemeshard.proto counters_sequenceshard.proto counters_sysview_processor.proto @@ -56,13 +56,13 @@ SRCS( counters.proto database_basic_sausage_metainfo.proto drivemodel.proto - export.proto + export.proto flat_tx_scheme.proto flat_scheme_op.proto health.proto hive.proto http_config.proto - import.proto + import.proto index_builder.proto issue_id.proto kesus.proto @@ -83,7 +83,7 @@ SRCS( node_limits.proto profiler.proto query_stats.proto - replication.proto + replication.proto resource_broker.proto scheme_log.proto scheme_type_metadata.proto @@ -126,8 +126,8 @@ SRCS( grpc_status_proxy.proto ydb_result_set_old.proto ydb_table_impl.proto - scheme_board.proto - scheme_board_mon.proto + scheme_board.proto + scheme_board_mon.proto sys_view.proto ) diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h index d9ca2273ab8..190233fdeee 100644 --- a/ydb/core/scheme/scheme_tabledefs.h +++ b/ydb/core/scheme/scheme_tabledefs.h @@ -28,42 +28,42 @@ struct TTableId { TTableId() = default; - // raw ctors - TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo, ui64 schemaVersion) + // raw ctors + TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo, ui64 schemaVersion) : PathId(ownerId, tableId) - , SysViewInfo(sysViewInfo) - , SchemaVersion(schemaVersion) + , SysViewInfo(sysViewInfo) + , SchemaVersion(schemaVersion) + {} + + TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo) + : TTableId(ownerId, tableId, sysViewInfo, 0) {} - TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo) - : TTableId(ownerId, tableId, sysViewInfo, 0) - {} - TTableId(ui64 ownerId, ui64 tableId, ui64 schemaVersion) - : TTableId(ownerId, tableId, TString(), schemaVersion) + : TTableId(ownerId, tableId, TString(), schemaVersion) + {} + + TTableId(ui64 ownerId, ui64 tableId) + : TTableId(ownerId, tableId, TString(), 0) + {} + + // ctors from TPathId + TTableId(const TPathId& pathId, const TString& sysViewInfo, ui64 schemaVersion) + : TTableId(pathId.OwnerId, pathId.LocalPathId, sysViewInfo, schemaVersion) + {} + + TTableId(const TPathId& pathId, const TString& sysViewInfo) + : TTableId(pathId, sysViewInfo, 0) {} - TTableId(ui64 ownerId, ui64 tableId) - : TTableId(ownerId, tableId, TString(), 0) + TTableId(const TPathId& pathId, ui64 schemaVersion) + : TTableId(pathId, TString(), schemaVersion) {} - // ctors from TPathId - TTableId(const TPathId& pathId, const TString& sysViewInfo, ui64 schemaVersion) - : TTableId(pathId.OwnerId, pathId.LocalPathId, sysViewInfo, schemaVersion) + TTableId(const TPathId& pathId) + : TTableId(pathId, TString(), 0) {} - TTableId(const TPathId& pathId, const TString& sysViewInfo) - : TTableId(pathId, sysViewInfo, 0) - {} - - TTableId(const TPathId& pathId, ui64 schemaVersion) - : TTableId(pathId, TString(), schemaVersion) - {} - - TTableId(const TPathId& pathId) - : TTableId(pathId, TString(), 0) - {} - explicit operator bool() const noexcept { return bool(PathId); } diff --git a/ydb/core/sys_view/common/events.h b/ydb/core/sys_view/common/events.h index 724bfeb175c..6e19aadc2b7 100644 --- a/ydb/core/sys_view/common/events.h +++ b/ydb/core/sys_view/common/events.h @@ -61,8 +61,8 @@ struct TEvSysView { EvSendDbCountersResponse, EvWatchDatabase, - EvUpdateTtlStats, - + EvUpdateTtlStats, + EvGetStorageStatsRequest, EvGetStorageStatsResponse, @@ -336,23 +336,23 @@ struct TEvSysView { : PathId(pathId) {} }; - - struct TEvUpdateTtlStats : public TEventLocal< - TEvUpdateTtlStats, - EvUpdateTtlStats> - { - TPathId DomainKey; - TPathId PathId; - TShardIdx ShardIdx; - - NKikimrSysView::TTtlStats Stats; - - TEvUpdateTtlStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx) - : DomainKey(domainKey) - , PathId(pathId) - , ShardIdx(shardIdx) - {} - }; + + struct TEvUpdateTtlStats : public TEventLocal< + TEvUpdateTtlStats, + EvUpdateTtlStats> + { + TPathId DomainKey; + TPathId PathId; + TShardIdx ShardIdx; + + NKikimrSysView::TTtlStats Stats; + + TEvUpdateTtlStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx) + : DomainKey(domainKey) + , PathId(pathId) + , ShardIdx(shardIdx) + {} + }; struct TEvGetStorageStatsRequest : TEventPB<TEvGetStorageStatsRequest, NKikimrSysView::TEvGetStorageStatsRequest, EvGetStorageStatsRequest> diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h index 5f39a7e7e11..298a8efbc03 100644 --- a/ydb/core/sys_view/common/scan_actor_base_impl.h +++ b/ydb/core/sys_view/common/scan_actor_base_impl.h @@ -233,7 +233,7 @@ private: return; } - SchemeShardId = entry.DomainInfo->ExtractSchemeShard(); + SchemeShardId = entry.DomainInfo->ExtractSchemeShard(); if (entry.DomainInfo->Params.HasSysViewProcessor()) { SysViewProcessorId = entry.DomainInfo->Params.GetSysViewProcessor(); diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index 559fe2c3975..f7a4f191e13 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -58,9 +58,9 @@ struct Schema : NIceDb::Schema { struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {}; - struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; - struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; - struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; + struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<OwnerId, PathId, PartIdx>; using TColumns = TableColumns< @@ -86,10 +86,10 @@ struct Schema : NIceDb::Schema { ImmediateTxCompleted, CoordinatedTxCompleted, TxRejectedByOverload, - TxRejectedByOutOfStorage, - LastTtlRunTime, - LastTtlRowsProcessed, - LastTtlRowsErased>; + TxRejectedByOutOfStorage, + LastTtlRunTime, + LastTtlRowsProcessed, + LastTtlRowsErased>; }; struct Nodes : Table<2> { diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp index 04076f0cf73..d6ea9ff73bd 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats.cpp +++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp @@ -31,7 +31,7 @@ public: hFunc(TEvSysView::TEvSetPartitioning, Handle); hFunc(TEvSysView::TEvRemoveTable, Handle); hFunc(TEvSysView::TEvSendPartitionStats, Handle); - hFunc(TEvSysView::TEvUpdateTtlStats, Handle); + hFunc(TEvSysView::TEvUpdateTtlStats, Handle); hFunc(TEvSysView::TEvGetPartitionStats, Handle); hFunc(TEvPrivate::TEvProcess, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); @@ -99,30 +99,30 @@ private: return; } - auto& oldStats = it->second.Partitions[shardIdx]; - auto& newStats = ev->Get()->Stats; - - if (oldStats.HasTtlStats()) { - newStats.MutableTtlStats()->Swap(oldStats.MutableTtlStats()); - } - - oldStats.Swap(&newStats); + auto& oldStats = it->second.Partitions[shardIdx]; + auto& newStats = ev->Get()->Stats; + + if (oldStats.HasTtlStats()) { + newStats.MutableTtlStats()->Swap(oldStats.MutableTtlStats()); + } + + oldStats.Swap(&newStats); + } + + void Handle(TEvSysView::TEvUpdateTtlStats::TPtr& ev) { + const auto& domainKey = ev->Get()->DomainKey; + const auto& pathId = ev->Get()->PathId; + const auto& shardIdx = ev->Get()->ShardIdx; + + auto& tables = DomainTables[domainKey]; + auto it = tables.find(pathId); + if (it == tables.end()) { + return; + } + + it->second.Partitions[shardIdx].MutableTtlStats()->Swap(&ev->Get()->Stats); } - void Handle(TEvSysView::TEvUpdateTtlStats::TPtr& ev) { - const auto& domainKey = ev->Get()->DomainKey; - const auto& pathId = ev->Get()->PathId; - const auto& shardIdx = ev->Get()->ShardIdx; - - auto& tables = DomainTables[domainKey]; - auto it = tables.find(pathId); - if (it == tables.end()) { - return; - } - - it->second.Partitions[shardIdx].MutableTtlStats()->Swap(&ev->Get()->Stats); - } - void Handle(TEvSysView::TEvGetPartitionStats::TPtr& ev) { if (PendingRequests.size() >= PendingRequestsLimit) { auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>(); @@ -490,15 +490,15 @@ private: insert({TSchema::TxRejectedByOutOfStorage::ColumnId, [] (const TPartitionStats& s) { return TCell::Make<ui64>(s.GetStats().GetTxRejectedBySpace()); }}); - insert({TSchema::LastTtlRunTime::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRunTime() * 1000) : TCell(); - }}); - insert({TSchema::LastTtlRowsProcessed::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsProcessed()) : TCell(); - }}); - insert({TSchema::LastTtlRowsErased::ColumnId, [] (const TPartitionStats& s) { - return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsErased()) : TCell(); - }}); + insert({TSchema::LastTtlRunTime::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRunTime() * 1000) : TCell(); + }}); + insert({TSchema::LastTtlRowsProcessed::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsProcessed()) : TCell(); + }}); + insert({TSchema::LastTtlRowsErased::ColumnId, [] (const TPartitionStats& s) { + return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsErased()) : TCell(); + }}); } }; static TExtractorsMap extractors; diff --git a/ydb/core/sys_view/processor/tx_init_schema.cpp b/ydb/core/sys_view/processor/tx_init_schema.cpp index 04f439e78b3..e0485d741f1 100644 --- a/ydb/core/sys_view/processor/tx_init_schema.cpp +++ b/ydb/core/sys_view/processor/tx_init_schema.cpp @@ -38,7 +38,7 @@ struct TSysViewProcessor::TTxInitSchema : public TTxBase { void Complete(const TActorContext& ctx) override { SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Complete"); - if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { + if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { SVLOG_D("[" << Self->TabletID() << "] tablet is offline"); Self->SignalTabletActive(ctx); Self->Become(&TThis::StateOffline); diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp index b3052090c26..918c9b781e6 100644 --- a/ydb/core/sys_view/query_stats/query_stats.cpp +++ b/ydb/core/sys_view/query_stats/query_stats.cpp @@ -179,7 +179,7 @@ private: } void StartScan() { - if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && this->SysViewProcessorId) { + if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && this->SysViewProcessorId) { RequestBatch(); } else { StartOldScan(); diff --git a/ydb/core/sys_view/service/sysview_service.cpp b/ydb/core/sys_view/service/sysview_service.cpp index ae8d417bb1a..1f569065999 100644 --- a/ydb/core/sys_view/service/sysview_service.cpp +++ b/ydb/core/sys_view/service/sysview_service.cpp @@ -26,7 +26,7 @@ void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery const TString& userSID, ui64 parametersSize, const TString& database, const NKikimrKqp::EQueryType type, ui64 requestUnits) { - if (!AppData()->FeatureFlags.GetEnableSystemViews()) { + if (!AppData()->FeatureFlags.GetEnableSystemViews()) { return; } @@ -339,7 +339,7 @@ public: ScanLimiter = MakeIntrusive<TScanLimiter>(ConcurrentScansLimit); - if (AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { + if (AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { IntervalEnd = GetNextIntervalEnd(); Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd)); } @@ -660,7 +660,7 @@ private: void Handle(TEvSysView::TEvGetIntervalMetricsRequest::TPtr& ev) { auto response = MakeHolder<TEvSysView::TEvGetIntervalMetricsResponse>(); - if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { + if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) { Send(ev->Sender, std::move(response), 0, ev->Cookie); return; } @@ -928,7 +928,7 @@ private: << ", query hash# " << stats->GetQueryTextHash() << ", cpu time# " << stats->GetTotalCpuTimeUs()); - if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && !database.empty()) { + if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && !database.empty()) { auto queryEnd = TInstant::MilliSeconds(stats->GetEndTimeMs()); if (queryEnd < IntervalEnd - TotalInterval) { return; diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index b68f5cf0e41..5e90eb59927 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -177,12 +177,12 @@ public: UNIT_ASSERT_GE(value.AsUint64(), expected); } - void Uint64LessOrEquals(ui64 expected) { - const auto& value = ExtractOptional(*RowIterator++); - UNIT_ASSERT(value.IsUint64()); - UNIT_ASSERT_LE(value.AsUint64(), expected); - } - + void Uint64LessOrEquals(ui64 expected) { + const auto& value = ExtractOptional(*RowIterator++); + UNIT_ASSERT(value.IsUint64()); + UNIT_ASSERT_LE(value.AsUint64(), expected); + } + void Int64(i64 expected) { const auto& value = ExtractOptional(*RowIterator++); UNIT_ASSERT(value.IsInt64()); @@ -316,7 +316,7 @@ Y_UNIT_TEST_SUITE(SystemView) { return; // table is currenty switched off TTestEnv env; - CreateTenantsAndTables(env, false); + CreateTenantsAndTables(env, false); TTableClient client(env.GetDriver()); { auto it = client.StreamExecuteScanQuery(R"( @@ -633,60 +633,60 @@ Y_UNIT_TEST_SUITE(SystemView) { check.Null(); // UserSID } - Y_UNIT_TEST(PartitionStatsTtlFields) { - TTestEnv env; - env.GetClient().CreateTable("/Root", R"( - Name: "Table0" - Columns { Name: "Key", Type: "Uint64" } - Columns { Name: "CreatedAt", Type: "Timestamp" } - KeyColumnNames: ["Key"] - TTLSettings { - Enabled { - ColumnName: "CreatedAt" - } - } - )"); - - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - NKqp::AssertSuccessResult(session.ExecuteDataQuery( - "REPLACE INTO `Root/Table0` (Key, CreatedAt) VALUES (0u, CAST(0 AS Timestamp));", - TTxControl::BeginTx().CommitTx() - ).GetValueSync()); - - // wait for conditional erase - for (size_t iter = 0; iter < 70; ++iter) { - auto result = session.ExecuteDataQuery( - "SELECT * FROM `Root/Table0`;", TTxControl::BeginTx().CommitTx() - ).ExtractValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - auto ysonString = FormatResultSetYson(result.GetResultSet(0)); - if (ysonString == "[[#]]") { - break; - } - - Sleep(TDuration::Seconds(1)); - } - - auto it = client.StreamExecuteScanQuery(R"( - SELECT - LastTtlRunTime, - LastTtlRowsProcessed, - LastTtlRowsErased - FROM `/Root/.sys/partition_stats`; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto ysonString = NKqp::StreamResultToYson(it); - - TYsonFieldChecker check(ysonString, 3); - - check.Uint64LessOrEquals(TInstant::Now().MicroSeconds()); // LastTtlRunTime - check.Uint64(1u); // LastTtlRowsProcessed - check.Uint64(1u); // LastTtlRowsErased - } - + Y_UNIT_TEST(PartitionStatsTtlFields) { + TTestEnv env; + env.GetClient().CreateTable("/Root", R"( + Name: "Table0" + Columns { Name: "Key", Type: "Uint64" } + Columns { Name: "CreatedAt", Type: "Timestamp" } + KeyColumnNames: ["Key"] + TTLSettings { + Enabled { + ColumnName: "CreatedAt" + } + } + )"); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + NKqp::AssertSuccessResult(session.ExecuteDataQuery( + "REPLACE INTO `Root/Table0` (Key, CreatedAt) VALUES (0u, CAST(0 AS Timestamp));", + TTxControl::BeginTx().CommitTx() + ).GetValueSync()); + + // wait for conditional erase + for (size_t iter = 0; iter < 70; ++iter) { + auto result = session.ExecuteDataQuery( + "SELECT * FROM `Root/Table0`;", TTxControl::BeginTx().CommitTx() + ).ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + auto ysonString = FormatResultSetYson(result.GetResultSet(0)); + if (ysonString == "[[#]]") { + break; + } + + Sleep(TDuration::Seconds(1)); + } + + auto it = client.StreamExecuteScanQuery(R"( + SELECT + LastTtlRunTime, + LastTtlRowsProcessed, + LastTtlRowsErased + FROM `/Root/.sys/partition_stats`; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto ysonString = NKqp::StreamResultToYson(it); + + TYsonFieldChecker check(ysonString, 3); + + check.Uint64LessOrEquals(TInstant::Now().MicroSeconds()); // LastTtlRunTime + check.Uint64(1u); // LastTtlRowsProcessed + check.Uint64(1u); // LastTtlRowsErased + } + Y_UNIT_TEST(PartitionStatsFields) { NDataShard::gDbStatsReportInterval = TDuration::Seconds(0); @@ -1181,7 +1181,7 @@ Y_UNIT_TEST_SUITE(SystemView) { const auto& columns = table.GetTableColumns(); const auto& keyColumns = table.GetPrimaryKeyColumns(); - UNIT_ASSERT_VALUES_EQUAL(columns.size(), 26); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 26); UNIT_ASSERT_STRINGS_EQUAL(columns[0].Name, "OwnerId"); UNIT_ASSERT_STRINGS_EQUAL(FormatType(columns[0].Type), "Uint64?"); @@ -1301,7 +1301,7 @@ Y_UNIT_TEST_SUITE(SystemView) { Y_UNIT_TEST(DescribeSystemFolder) { TTestEnv env; - CreateTenantsAndTables(env, false); + CreateTenantsAndTables(env, false); TSchemeClient schemeClient(env.GetDriver()); { @@ -1324,7 +1324,7 @@ Y_UNIT_TEST_SUITE(SystemView) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); auto entry = result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Tenant1"); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Tenant1"); UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::SubDomain); auto children = result.GetChildren(); @@ -1377,7 +1377,7 @@ Y_UNIT_TEST_SUITE(SystemView) { Y_UNIT_TEST(DescribeAccessDenied) { TTestEnv env; - CreateTenantsAndTables(env, false); + CreateTenantsAndTables(env, false); auto driverConfig = TDriverConfig() .SetEndpoint(env.GetEndpoint()) diff --git a/ydb/core/tablet/resource_broker.cpp b/ydb/core/tablet/resource_broker.cpp index 1162ae020b1..fff30b26010 100644 --- a/ydb/core/tablet/resource_broker.cpp +++ b/ydb/core/tablet/resource_broker.cpp @@ -1325,11 +1325,11 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() queue->MutableLimit()->SetCpu(2); queue = config.AddQueues(); - queue->SetName("queue_restore"); - queue->SetWeight(100); - queue->MutableLimit()->SetCpu(10); - - queue = config.AddQueues(); + queue->SetName("queue_restore"); + queue->SetWeight(100); + queue->MutableLimit()->SetCpu(10); + + queue = config.AddQueues(); queue->SetName(NLocalDb::KqpResourceManagerQueue); queue->SetWeight(30); queue->MutableLimit()->SetCpu(KqpRmQueueCPU); @@ -1411,11 +1411,11 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() task->SetDefaultDuration(TDuration::Minutes(5).GetValue()); task = config.AddTasks(); - task->SetName("restore"); - task->SetQueueName("queue_restore"); - task->SetDefaultDuration(TDuration::Minutes(5).GetValue()); - - task = config.AddTasks(); + task->SetName("restore"); + task->SetQueueName("queue_restore"); + task->SetDefaultDuration(TDuration::Minutes(5).GetValue()); + + task = config.AddTasks(); task->SetName(NLocalDb::KqpResourceManagerTaskName); task->SetQueueName(NLocalDb::KqpResourceManagerQueue); task->SetDefaultDuration(TDuration::Minutes(10).GetValue()); diff --git a/ydb/core/tablet/tablet_counters.h b/ydb/core/tablet/tablet_counters.h index 110f48fdf14..a8cc27e9eda 100644 --- a/ydb/core/tablet/tablet_counters.h +++ b/ydb/core/tablet/tablet_counters.h @@ -166,14 +166,14 @@ public: return *this; } - TTabletPercentileCounter& DecrementFor(ui64 what) { - Y_VERIFY(Integral); - ui32 index = FindSlot(what); - Y_VERIFY_DEBUG(Values[index] > 0); - Values[index] -= 1; - return *this; - } - + TTabletPercentileCounter& DecrementFor(ui64 what) { + Y_VERIFY(Integral); + ui32 index = FindSlot(what); + Y_VERIFY_DEBUG(Values[index] > 0); + Values[index] -= 1; + return *this; + } + ui32 GetRangeCount() const { return RangeCount; } diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 93be3b86dee..1885b26e09c 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -576,9 +576,9 @@ public: } QuietTabletCounters.erase(tabletID); - - TString tabletIdStr = Sprintf("%" PRIu64, tabletID); - Counters->RemoveSubgroup("tabletid", tabletIdStr.data()); + + TString tabletIdStr = Sprintf("%" PRIu64, tabletID); + Counters->RemoveSubgroup("tabletid", tabletIdStr.data()); } void Query(const NKikimrTabletCountersAggregator::TEvTabletCountersRequest& request, NKikimrTabletCountersAggregator::TEvTabletCountersResponse& response) { diff --git a/ydb/core/tablet/tablet_sys.cpp b/ydb/core/tablet/tablet_sys.cpp index 8dba39c3091..44bfac7b2d6 100644 --- a/ydb/core/tablet/tablet_sys.cpp +++ b/ydb/core/tablet/tablet_sys.cpp @@ -1,18 +1,18 @@ #include "tablet_sys.h" -#include "tablet_tracing_signals.h" - +#include "tablet_tracing_signals.h" + #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/base/hive.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/protos/services.pb.h> -#include <library/cpp/actors/core/log.h> - +#include <library/cpp/actors/core/log.h> + #include <util/generic/deque.h> #include <util/generic/hash.h> #include <util/generic/queue.h> #include <util/generic/set.h> -#include <util/stream/str.h> +#include <util/stream/str.h> #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR #error log macro definition clash diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 3d2f29bfa0a..27a260bcb27 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -617,12 +617,12 @@ struct TNull { } }; -enum class EMaterializationMode { - All, - Existing, - NonExisting, -}; - +enum class EMaterializationMode { + All, + Existing, + NonExisting, +}; + struct Schema { template <typename T> struct Precharger { @@ -1850,10 +1850,10 @@ struct Schema { static bool Precharge(TToughDb& database) { return SchemaTables<Type>::Precharge(database) & SchemaTables<Types...>::Precharge(database); } - - static void Materialize(TToughDb& database, EMaterializationMode mode = EMaterializationMode::All) { - SchemaTables<Type>::Materialize(database, mode); - SchemaTables<Types...>::Materialize(database, mode); + + static void Materialize(TToughDb& database, EMaterializationMode mode = EMaterializationMode::All) { + SchemaTables<Type>::Materialize(database, mode); + SchemaTables<Types...>::Materialize(database, mode); } static void Cleanup(TToughDb& database) { @@ -1876,22 +1876,22 @@ struct Schema { return typename Type::TKey::template Selector<Type>(database).Precharge(); } - static void Materialize(TToughDb& database, EMaterializationMode mode = EMaterializationMode::All) { - switch (mode) { - case EMaterializationMode::All: - break; - case EMaterializationMode::Existing: - if (!database.GetScheme().GetTableInfo(Type::TableId)) { - return; - } - break; - case EMaterializationMode::NonExisting: - if (database.GetScheme().GetTableInfo(Type::TableId)) { - return; - } - break; - } - + static void Materialize(TToughDb& database, EMaterializationMode mode = EMaterializationMode::All) { + switch (mode) { + case EMaterializationMode::All: + break; + case EMaterializationMode::Existing: + if (!database.GetScheme().GetTableInfo(Type::TableId)) { + return; + } + break; + case EMaterializationMode::NonExisting: + if (database.GetScheme().GetTableInfo(Type::TableId)) { + return; + } + break; + } + database.Alter().AddTable(GetTableName(TypeName<Type>()), Type::TableId); Type::TColumns::Materialize(database); Type::TKey::Materialize(database); @@ -1942,11 +1942,11 @@ public: template <typename TableType> typename TableType::TKey::template Selector<TableType> Table() { return Database; } - template <typename TableType> - bool HaveTable() { - return Database.GetScheme().GetTableInfo(TableType::TableId); - } - + template <typename TableType> + bool HaveTable() { + return Database.GetScheme().GetTableInfo(TableType::TableId); + } + template <typename SchemaType> bool Precharge() { return SchemaType::TTables::Precharge(Database); @@ -1955,20 +1955,20 @@ public: template <typename SchemaType> void Materialize() { SchemaType::TSettings::Materialize(Database); - SchemaType::TTables::Materialize(Database, EMaterializationMode::All); + SchemaType::TTables::Materialize(Database, EMaterializationMode::All); + } + + template <typename SchemaType> + void MaterializeExisting() { + SchemaType::TTables::Materialize(Database, EMaterializationMode::Existing); + } + + template <typename SchemaType> + void MaterializeNonExisting() { + SchemaType::TTables::Materialize(Database, EMaterializationMode::NonExisting); } template <typename SchemaType> - void MaterializeExisting() { - SchemaType::TTables::Materialize(Database, EMaterializationMode::Existing); - } - - template <typename SchemaType> - void MaterializeNonExisting() { - SchemaType::TTables::Materialize(Database, EMaterializationMode::NonExisting); - } - - template <typename SchemaType> void Cleanup() { SchemaType::TTables::Cleanup(Database); diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 3c1ed9f4278..59afc462158 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -22,9 +22,9 @@ #include "logic_alter_main.h" #include "flat_abi_evol.h" #include "probes.h" -#include "shared_sausagecache.h" +#include "shared_sausagecache.h" #include "util_fmt_desc.h" - + #include <ydb/core/base/appdata.h> #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/base/hive.h> @@ -34,10 +34,10 @@ #include <ydb/core/tablet/tablet_counters_aggregator.h> #include <ydb/core/util/yverify_stream.h> -#include <library/cpp/monlib/service/pages/templates.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/xrange.h> +#include <library/cpp/monlib/service/pages/templates.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <util/generic/xrange.h> #include <util/generic/ymath.h> namespace NKikimr { diff --git a/ydb/core/tablet_flat/test/libs/exec/runner.h b/ydb/core/tablet_flat/test/libs/exec/runner.h index cddca5d32bb..3b58ab814b4 100644 --- a/ydb/core/tablet_flat/test/libs/exec/runner.h +++ b/ydb/core/tablet_flat/test/libs/exec/runner.h @@ -148,7 +148,7 @@ namespace NFake { } { - auto *actor = CreateStateStorageProxy(info, nullptr, nullptr); + auto *actor = CreateStateStorageProxy(info, nullptr, nullptr); AddService(MakeStateStorageProxyID(0), actor, TMailboxType::Revolving); } diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index 773e8cd298a..63f0436bff7 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -125,7 +125,7 @@ namespace NActors { nodeAppData->Counters = node->DynamicCounters; nodeAppData->PollerThreads = node->Poller; nodeAppData->StreamingConfig.SetEnableOutputStreams(true); - nodeAppData->PQConfig = app0->PQConfig; + nodeAppData->PQConfig = app0->PQConfig; nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig); nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig); nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling; @@ -133,9 +133,9 @@ namespace NActors { nodeAppData->CompactionConfig = app0->CompactionConfig; nodeAppData->HiveConfig = app0->HiveConfig; nodeAppData->DataShardConfig = app0->DataShardConfig; - nodeAppData->MeteringConfig = app0->MeteringConfig; + nodeAppData->MeteringConfig = app0->MeteringConfig; nodeAppData->EnableMvccSnapshotWithLegacyDomainRoot = app0->EnableMvccSnapshotWithLegacyDomainRoot; - nodeAppData->IoContextFactory = app0->IoContextFactory; + nodeAppData->IoContextFactory = app0->IoContextFactory; if (KeyConfigGenerator) { nodeAppData->KeyConfig = KeyConfigGenerator(nodeIndex); } else { diff --git a/ydb/core/testlib/basics/appdata.h b/ydb/core/testlib/basics/appdata.h index e4b99def370..409869161f5 100644 --- a/ydb/core/testlib/basics/appdata.h +++ b/ydb/core/testlib/basics/appdata.h @@ -1,7 +1,7 @@ #pragma once -#include "feature_flags.h" - +#include "feature_flags.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/channel_profiles.h> #include <ydb/core/base/domain.h> @@ -17,24 +17,24 @@ namespace NKikimr { // Split this factory class TDataShardExportFactory : public NKikimr::NDataShard::IExportFactory { public: - NKikimr::NDataShard::IExport* CreateExportToYt(bool useTypeV3) const override { - Y_UNUSED(useTypeV3); + NKikimr::NDataShard::IExport* CreateExportToYt(bool useTypeV3) const override { + Y_UNUSED(useTypeV3); return nullptr; } - NKikimr::NDataShard::IExport* CreateExportToS3() const override { - #ifndef KIKIMR_DISABLE_S3_OPS - return new NKikimr::NDataShard::TS3Export(); - #else - return nullptr; - #endif + NKikimr::NDataShard::IExport* CreateExportToS3() const override { + #ifndef KIKIMR_DISABLE_S3_OPS + return new NKikimr::NDataShard::TS3Export(); + #else + return nullptr; + #endif } void Shutdown() override { } }; - struct TAppPrepare : public TTestFeatureFlagsHolder<TAppPrepare> { + struct TAppPrepare : public TTestFeatureFlagsHolder<TAppPrepare> { struct TMine : public NActors::IDestructable { TIntrusivePtr<NScheme::TTypeRegistry> Types; TIntrusivePtr<NMiniKQL::IFunctionRegistry> Funcs; @@ -78,8 +78,8 @@ namespace NKikimr { TString NetDataSourceUrl; NKikimrConfig::THiveConfig HiveConfig; NKikimrConfig::TDataShardConfig DataShardConfig; - NKikimrConfig::TMeteringConfig MeteringConfig; - NKikimrPQ::TPQConfig PQConfig; + NKikimrConfig::TMeteringConfig MeteringConfig; + NKikimrPQ::TPQConfig PQConfig; private: TAutoPtr<TMine> Mine; diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index aa6ba5e48f8..c78752dad18 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -1,53 +1,53 @@ -#pragma once - +#pragma once + #include <ydb/core/base/feature_flags.h> - -namespace NKikimr { - -template <typename TDerived> -class TTestFeatureFlagsHolder { -public: - TFeatureFlags FeatureFlags; - - #define FEATURE_FLAG_SETTER(name) \ - TDerived& Set##name(std::optional<bool> value) { \ - if (value) { \ - FeatureFlags.Set##name(*value); \ - } \ - return *static_cast<TDerived*>(this); \ - } - - FEATURE_FLAG_SETTER(AllowYdbRequestsWithoutDatabase) - FEATURE_FLAG_SETTER(EnableSystemViews) - FEATURE_FLAG_SETTER(CheckDatabaseAccessPermission) - FEATURE_FLAG_SETTER(EnablePersistentQueryStats) - FEATURE_FLAG_SETTER(EnablePersistentPartitionStats) - FEATURE_FLAG_SETTER(AllowUpdateChannelsBindingOfSolomonPartitions) - FEATURE_FLAG_SETTER(EnableDataColumnForIndexTable) - FEATURE_FLAG_SETTER(EnableClockGettimeForUserCpuAccounting) - FEATURE_FLAG_SETTER(EnableAsyncIndexes) - FEATURE_FLAG_SETTER(EnableOlapSchemaOperations) - FEATURE_FLAG_SETTER(EnableMvccSnapshotReads) - FEATURE_FLAG_SETTER(EnableSchemeTransactionsAtSchemeShard) - FEATURE_FLAG_SETTER(EnableBackgroundCompaction) - FEATURE_FLAG_SETTER(EnableBackgroundCompactionServerless) - FEATURE_FLAG_SETTER(EnableNotNullColumns) - FEATURE_FLAG_SETTER(EnableTtlOnAsyncIndexedTables) - FEATURE_FLAG_SETTER(EnableBulkUpsertToAsyncIndexedTables) - - TDerived& SetEnableMvcc(std::optional<bool> value) { - if (value) { - if (*value) { - FeatureFlags.SetEnableMvcc(NKikimrConfig::TFeatureFlags::VALUE_TRUE); - } else { - FeatureFlags.SetEnableMvcc(NKikimrConfig::TFeatureFlags::VALUE_FALSE); - } - } - - return *static_cast<TDerived*>(this); - } - - #undef FEATURE_FLAG_SETTER -}; - -} // NKikimr + +namespace NKikimr { + +template <typename TDerived> +class TTestFeatureFlagsHolder { +public: + TFeatureFlags FeatureFlags; + + #define FEATURE_FLAG_SETTER(name) \ + TDerived& Set##name(std::optional<bool> value) { \ + if (value) { \ + FeatureFlags.Set##name(*value); \ + } \ + return *static_cast<TDerived*>(this); \ + } + + FEATURE_FLAG_SETTER(AllowYdbRequestsWithoutDatabase) + FEATURE_FLAG_SETTER(EnableSystemViews) + FEATURE_FLAG_SETTER(CheckDatabaseAccessPermission) + FEATURE_FLAG_SETTER(EnablePersistentQueryStats) + FEATURE_FLAG_SETTER(EnablePersistentPartitionStats) + FEATURE_FLAG_SETTER(AllowUpdateChannelsBindingOfSolomonPartitions) + FEATURE_FLAG_SETTER(EnableDataColumnForIndexTable) + FEATURE_FLAG_SETTER(EnableClockGettimeForUserCpuAccounting) + FEATURE_FLAG_SETTER(EnableAsyncIndexes) + FEATURE_FLAG_SETTER(EnableOlapSchemaOperations) + FEATURE_FLAG_SETTER(EnableMvccSnapshotReads) + FEATURE_FLAG_SETTER(EnableSchemeTransactionsAtSchemeShard) + FEATURE_FLAG_SETTER(EnableBackgroundCompaction) + FEATURE_FLAG_SETTER(EnableBackgroundCompactionServerless) + FEATURE_FLAG_SETTER(EnableNotNullColumns) + FEATURE_FLAG_SETTER(EnableTtlOnAsyncIndexedTables) + FEATURE_FLAG_SETTER(EnableBulkUpsertToAsyncIndexedTables) + + TDerived& SetEnableMvcc(std::optional<bool> value) { + if (value) { + if (*value) { + FeatureFlags.SetEnableMvcc(NKikimrConfig::TFeatureFlags::VALUE_TRUE); + } else { + FeatureFlags.SetEnableMvcc(NKikimrConfig::TFeatureFlags::VALUE_FALSE); + } + } + + return *static_cast<TDerived*>(this); + } + + #undef FEATURE_FLAG_SETTER +}; + +} // NKikimr diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp index 4e23d87fdc7..b4585ad96c6 100644 --- a/ydb/core/testlib/basics/services.cpp +++ b/ydb/core/testlib/basics/services.cpp @@ -47,12 +47,12 @@ namespace NPDisk { void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root) { - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); cacheConfig->Roots.emplace_back(1, TTestTxConfig::SchemeShard, root); cacheConfig->Counters = new NMonitoring::TDynamicCounters(); - runtime.AddLocalService(MakeSchemeCacheID(), - TActorSetupCmd(CreateSchemeBoardSchemeCache(cacheConfig.Get()), TMailboxType::Revolving, 0), nodeIndex); + runtime.AddLocalService(MakeSchemeCacheID(), + TActorSetupCmd(CreateSchemeBoardSchemeCache(cacheConfig.Get()), TMailboxType::Revolving, 0), nodeIndex); } void SetupTabletResolver(TTestActorRuntime& runtime, ui32 nodeIndex) @@ -150,20 +150,20 @@ namespace NPDisk { nodeIndex); } - template<size_t N> + template<size_t N> static TIntrusivePtr<TStateStorageInfo> GenerateStateStorageInfo(const TActorId (&replicas)[N], ui64 stateStorageGroup) - { - TIntrusivePtr<TStateStorageInfo> info(new TStateStorageInfo()); - info->StateStorageGroup = stateStorageGroup; + { + TIntrusivePtr<TStateStorageInfo> info(new TStateStorageInfo()); + info->StateStorageGroup = stateStorageGroup; info->NToSelect = N; info->Rings.resize(N); - for (size_t i = 0; i < N; ++i) { + for (size_t i = 0; i < N; ++i) { info->Rings[i].Replicas.push_back(replicas[i]); - } - - return info; - } - + } + + return info; + } + static TActorId MakeBoardReplicaID( const ui32 node, const ui64 stateStorageGroup, @@ -190,23 +190,23 @@ namespace NPDisk { }; const TActorId sbreplicas[3] = { - MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 0), - MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 1), - MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 2), - }; - + MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 0), + MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 1), + MakeSchemeBoardReplicaID(runtime.GetNodeId(0), stateStorageGroup, 2), + }; + const TActorId ssproxy = MakeStateStorageProxyID(stateStorageGroup); - auto ssInfo = GenerateStateStorageInfo(ssreplicas, stateStorageGroup); - auto sbInfo = GenerateStateStorageInfo(sbreplicas, stateStorageGroup); + auto ssInfo = GenerateStateStorageInfo(ssreplicas, stateStorageGroup); + auto sbInfo = GenerateStateStorageInfo(sbreplicas, stateStorageGroup); auto bInfo = GenerateStateStorageInfo(breplicas, stateStorageGroup); if (!firstNode || nodeIndex == 0) { for (ui32 i = 0; i < 3; ++i) { runtime.AddLocalService(ssreplicas[i], TActorSetupCmd(CreateStateStorageReplica(ssInfo.Get(), i), TMailboxType::Revolving, 0), nodeIndex); - runtime.AddLocalService(sbreplicas[i], - TActorSetupCmd(CreateSchemeBoardReplica(sbInfo.Get(), i), TMailboxType::Revolving, 0), nodeIndex); + runtime.AddLocalService(sbreplicas[i], + TActorSetupCmd(CreateSchemeBoardReplica(sbInfo.Get(), i), TMailboxType::Revolving, 0), nodeIndex); runtime.AddLocalService(breplicas[i], TActorSetupCmd(CreateStateStorageBoardReplica(bInfo.Get(), i), TMailboxType::Revolving, 0), nodeIndex); } diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index 96390c79379..a853040b932 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -20,7 +20,7 @@ #include <ydb/core/tx/tx_allocator/txallocator.h> #include <ydb/core/tx/coordinator/coordinator.h> #include <ydb/core/tx/mediator/mediator.h> -#include <ydb/core/tx/replication/controller/controller.h> +#include <ydb/core/tx/replication/controller/controller.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/sequenceshard/sequenceshard.h> #include <ydb/core/tx/time_cast/time_cast.h> @@ -1187,8 +1187,8 @@ namespace NKikimr { bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); } else if (type == defaultTabletTypes.SequenceShard) { bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure); - } else if (type == defaultTabletTypes.ReplicationController) { - bootstrapperActorId = Boot(ctx, type, &NReplication::CreateController, DataGroupErasure); + } else if (type == defaultTabletTypes.ReplicationController) { + bootstrapperActorId = Boot(ctx, type, &NReplication::CreateController, DataGroupErasure); } else { status = NKikimrProto::ERROR; } diff --git a/ydb/core/testlib/tenant_helpers.h b/ydb/core/testlib/tenant_helpers.h index 1277aa42e55..209b73baa0e 100644 --- a/ydb/core/testlib/tenant_helpers.h +++ b/ydb/core/testlib/tenant_helpers.h @@ -65,63 +65,63 @@ void CollectSlots(THashMap<std::pair<TString, TString>, TSlotState> &slots, CollectSlots(slots, args...); } -struct TCreateTenantRequest { - using TSelf = TCreateTenantRequest; - using TAttrsCont = TVector<std::pair<TString, TString>>; - using TPoolsCont = TVector<TPoolAllocation>; - using TSlotsCont = TVector<TSlotRequest>; - - enum class EType { - Unspecified, - Common, - Shared, - Serverless, - }; - - // All types - TString Path; - EType Type; - TAttrsCont Attrs; - // Common & Shared - TPoolsCont Pools; - TSlotsCont Slots; - // Serverless - TString SharedDbPath; +struct TCreateTenantRequest { + using TSelf = TCreateTenantRequest; + using TAttrsCont = TVector<std::pair<TString, TString>>; + using TPoolsCont = TVector<TPoolAllocation>; + using TSlotsCont = TVector<TSlotRequest>; + + enum class EType { + Unspecified, + Common, + Shared, + Serverless, + }; + + // All types + TString Path; + EType Type; + TAttrsCont Attrs; + // Common & Shared + TPoolsCont Pools; + TSlotsCont Slots; + // Serverless + TString SharedDbPath; ui32 PlanResolution = 0; - - TCreateTenantRequest() = delete; - - explicit TCreateTenantRequest(const TString& path, EType type = EType::Unspecified) - : Path(path) - , Type(type) - { - } - - TSelf& WithAttrs(const TAttrsCont& attrs) { - Attrs = attrs; - return *this; - } - - TSelf& WithPools(const TPoolsCont& pools) { - if (Type == EType::Unspecified) { - Type = EType::Common; - } - - UNIT_ASSERT(Type == EType::Common || Type == EType::Shared); - Pools = pools; - return *this; - } - - TSelf& WithSlots(const TSlotsCont& slots) { - if (Type == EType::Unspecified) { - Type = EType::Common; - } - - UNIT_ASSERT(Type == EType::Common || Type == EType::Shared); - Slots = slots; - return *this; - } - + + TCreateTenantRequest() = delete; + + explicit TCreateTenantRequest(const TString& path, EType type = EType::Unspecified) + : Path(path) + , Type(type) + { + } + + TSelf& WithAttrs(const TAttrsCont& attrs) { + Attrs = attrs; + return *this; + } + + TSelf& WithPools(const TPoolsCont& pools) { + if (Type == EType::Unspecified) { + Type = EType::Common; + } + + UNIT_ASSERT(Type == EType::Common || Type == EType::Shared); + Pools = pools; + return *this; + } + + TSelf& WithSlots(const TSlotsCont& slots) { + if (Type == EType::Unspecified) { + Type = EType::Common; + } + + UNIT_ASSERT(Type == EType::Common || Type == EType::Shared); + Slots = slots; + return *this; + } + TSelf& WithSlots(const TString& type, const TString& zone, ui64 count) { if (Type == EType::Unspecified) { Type = EType::Common; @@ -132,45 +132,45 @@ struct TCreateTenantRequest { return *this; } - TSelf& WithSharedDbPath(const TString& path) { - if (Type == EType::Unspecified) { - Type = EType::Serverless; - } - - UNIT_ASSERT(Type == EType::Serverless); - SharedDbPath = path; - return *this; - } + TSelf& WithSharedDbPath(const TString& path) { + if (Type == EType::Unspecified) { + Type = EType::Serverless; + } + + UNIT_ASSERT(Type == EType::Serverless); + SharedDbPath = path; + return *this; + } TSelf& WithPlanResolution(ui32 planResolution) { PlanResolution = planResolution; return *this; } -}; - -inline void CheckCreateTenant(TTenantTestRuntime &runtime, +}; + +inline void CheckCreateTenant(TTenantTestRuntime &runtime, const TString &token, Ydb::StatusIds::StatusCode code, - const TCreateTenantRequest &request) + const TCreateTenantRequest &request) { - using EType = TCreateTenantRequest::EType; + using EType = TCreateTenantRequest::EType; auto *event = new NConsole::TEvConsole::TEvCreateTenantRequest; - event->Record.MutableRequest()->set_path(request.Path); - - auto *resources = request.Type == EType::Shared - ? event->Record.MutableRequest()->mutable_shared_resources() - : event->Record.MutableRequest()->mutable_resources(); - - for (auto &req : request.Slots) { - auto &unit = *resources->add_computational_units(); + event->Record.MutableRequest()->set_path(request.Path); + + auto *resources = request.Type == EType::Shared + ? event->Record.MutableRequest()->mutable_shared_resources() + : event->Record.MutableRequest()->mutable_resources(); + + for (auto &req : request.Slots) { + auto &unit = *resources->add_computational_units(); unit.set_unit_kind(req.Type); unit.set_availability_zone(req.Zone); unit.set_count(req.Count); } - for (auto &pool : request.Pools) { - auto &unit = *resources->add_storage_units(); + for (auto &pool : request.Pools) { + auto &unit = *resources->add_storage_units(); unit.set_unit_kind(pool.PoolType); unit.set_count(pool.PoolSize); } @@ -182,10 +182,10 @@ inline void CheckCreateTenant(TTenantTestRuntime &runtime, (*event->Record.MutableRequest()->mutable_attributes())[key] = value; } - if (request.Type == EType::Serverless) { - event->Record.MutableRequest()->mutable_serverless_resources()->set_shared_database_path(request.SharedDbPath); - } - + if (request.Type == EType::Serverless) { + event->Record.MutableRequest()->mutable_serverless_resources()->set_shared_database_path(request.SharedDbPath); + } + if (request.PlanResolution) { event->Record.MutableRequest()->mutable_options()->set_plan_resolution(request.PlanResolution); } @@ -217,40 +217,40 @@ inline void CheckCreateTenant(TTenantTestRuntime &runtime, } } -inline void CheckCreateTenant(TTenantTestRuntime &runtime, - Ydb::StatusIds::StatusCode code, - const TCreateTenantRequest &request) -{ - CheckCreateTenant(runtime, "", code, request); -} - +inline void CheckCreateTenant(TTenantTestRuntime &runtime, + Ydb::StatusIds::StatusCode code, + const TCreateTenantRequest &request) +{ + CheckCreateTenant(runtime, "", code, request); +} + +template <typename ...Ts> +void CheckCreateTenant(TTenantTestRuntime &runtime, + const TString &path, + const TString &token, + Ydb::StatusIds::StatusCode code, + TVector<TPoolAllocation> pools, + const TVector<std::pair<TString, TString>> &attrs, + Ts... args) +{ + TVector<TSlotRequest> slots; + CollectSlots(slots, args...); + + CheckCreateTenant(runtime, token, code, + TCreateTenantRequest(path) + .WithSlots(slots) + .WithPools(pools) + .WithAttrs(attrs)); +} + template <typename ...Ts> void CheckCreateTenant(TTenantTestRuntime &runtime, const TString &path, - const TString &token, Ydb::StatusIds::StatusCode code, TVector<TPoolAllocation> pools, const TVector<std::pair<TString, TString>> &attrs, Ts... args) { - TVector<TSlotRequest> slots; - CollectSlots(slots, args...); - - CheckCreateTenant(runtime, token, code, - TCreateTenantRequest(path) - .WithSlots(slots) - .WithPools(pools) - .WithAttrs(attrs)); -} - -template <typename ...Ts> -void CheckCreateTenant(TTenantTestRuntime &runtime, - const TString &path, - Ydb::StatusIds::StatusCode code, - TVector<TPoolAllocation> pools, - const TVector<std::pair<TString, TString>> &attrs, - Ts... args) -{ CheckCreateTenant(runtime, path, "", code, pools, attrs, args...); } diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp index 2ce722357d4..974cc166d0d 100644 --- a/ydb/core/testlib/tenant_runtime.cpp +++ b/ydb/core/testlib/tenant_runtime.cpp @@ -16,7 +16,7 @@ #include <ydb/core/tx/long_tx_service/public/events.h> #include <ydb/core/tx/long_tx_service/long_tx_service.h> #include <ydb/core/tx/mediator/mediator.h> -#include <ydb/core/tx/replication/controller/controller.h> +#include <ydb/core/tx/replication/controller/controller.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/sequenceproxy/sequenceproxy.h> #include <ydb/core/tx/sequenceshard/sequenceshard.h> @@ -435,8 +435,8 @@ class TFakeHive : public TActor<TFakeHive>, public TTabletExecutedFlat { bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure); } else if (type == defaultTabletTypes.SequenceShard) { bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure); - } else if (type == defaultTabletTypes.ReplicationController) { - bootstrapperActorId = Boot(ctx, type, &NReplication::CreateController, DataGroupErasure); + } else if (type == defaultTabletTypes.ReplicationController) { + bootstrapperActorId = Boot(ctx, type, &NReplication::CreateController, DataGroupErasure); } else { status = NKikimrProto::ERROR; } @@ -773,7 +773,7 @@ void TTenantTestRuntime::CreateTenantPool(ui32 nodeIndex) CreateTenantPool(nodeIndex, Config.Nodes[nodeIndex].TenantPoolConfig); } -void TTenantTestRuntime::Setup(bool createTenantPools) +void TTenantTestRuntime::Setup(bool createTenantPools) { if (ENABLE_DETAILED_LOG) { SetLogPriority(NKikimrServices::LOCAL, NLog::PRI_DEBUG); @@ -806,7 +806,7 @@ void TTenantTestRuntime::Setup(bool createTenantPools) app.FeatureFlags = Extension.GetFeatureFlags(); app.ClearDomainsAndHive(); - + ui32 planResolution = 500; // Add domains info. for (ui32 i = 0; i < Config.Domains.size(); ++i) { @@ -1001,14 +1001,14 @@ void TTenantTestRuntime::Setup(bool createTenantPools) // Create Tenant Slot Pools Y_VERIFY(GetNodeCount() >= Config.Nodes.size()); TMultiSet<std::pair<TString, TEvLocal::TEvTenantStatus::EStatus>> statuses; - - if (createTenantPools) { - for (size_t i = 0; i< Config.Nodes.size(); ++i) { - auto &poolConfig = Config.Nodes[i].TenantPoolConfig; - for (auto &slot : poolConfig.StaticSlots) - statuses.insert(std::make_pair(CanonizePath(slot.Tenant), TEvLocal::TEvTenantStatus::STARTED)); - CreateTenantPool(i); - } + + if (createTenantPools) { + for (size_t i = 0; i< Config.Nodes.size(); ++i) { + auto &poolConfig = Config.Nodes[i].TenantPoolConfig; + for (auto &slot : poolConfig.StaticSlots) + statuses.insert(std::make_pair(CanonizePath(slot.Tenant), TEvLocal::TEvTenantStatus::STARTED)); + CreateTenantPool(i); + } } // Create other local services @@ -1109,7 +1109,7 @@ void TTenantTestRuntime::Setup(bool createTenantPools) EnableScheduleForActor(actorId, true); // Wait until Tenant Slot Broker gets all statuses. - if (createTenantPools && !Config.FakeTenantSlotBroker) { + if (createTenantPools && !Config.FakeTenantSlotBroker) { TDispatchOptions options; options.FinalEvents.emplace_back(TWaitTenantSlotBrokerInitialization(Config.Nodes.size())); DispatchEvents(options); @@ -1119,13 +1119,13 @@ void TTenantTestRuntime::Setup(bool createTenantPools) TTenantTestRuntime::TTenantTestRuntime(const TTenantTestConfig &config, const NKikimrConfig::TAppConfig &extension, - bool createTenantPools) + bool createTenantPools) : TTestBasicRuntime(config.Nodes.size(), config.DataCenterCount, false) , Config(config) , Extension(extension) { Extension.MutableFeatureFlags()->SetEnableExternalHive(false); - Setup(createTenantPools); + Setup(createTenantPools); } void TTenantTestRuntime::WaitForHiveState(const TVector<TEvTest::TEvWaitHiveState::TClientInfo> &state) diff --git a/ydb/core/testlib/tenant_runtime.h b/ydb/core/testlib/tenant_runtime.h index 263f8fabfae..f98c1441af3 100644 --- a/ydb/core/testlib/tenant_runtime.h +++ b/ydb/core/testlib/tenant_runtime.h @@ -192,12 +192,12 @@ bool IsTabletActiveEvent(IEventHandle& ev); class TTenantTestRuntime : public TTestBasicRuntime { private: - void Setup(bool createTenantPools = true); + void Setup(bool createTenantPools = true); public: TTenantTestRuntime(const TTenantTestConfig &config, const NKikimrConfig::TAppConfig &extension = {}, - bool createTenantPools = true); + bool createTenantPools = true); void CreateTenantPool(ui32 nodeIndex); void CreateTenantPool(ui32 nodeIndex, const TTenantTestConfig::TTenantPoolConfig &config); diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 604cec0b8f6..d4907c26f1e 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -5,7 +5,7 @@ #include <ydb/core/base/hive.h> #include <ydb/public/lib/base/msgbus.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> -#include <ydb/services/auth/grpc_service.h> +#include <ydb/services/auth/grpc_service.h> #include <ydb/services/yq/grpc_service.h> #include <ydb/services/yq/private_grpc.h> #include <ydb/services/cms/grpc_service.h> @@ -53,7 +53,7 @@ #include <ydb/core/tx/long_tx_service/public/events.h> #include <ydb/core/tx/long_tx_service/long_tx_service.h> #include <ydb/core/tx/mediator/mediator.h> -#include <ydb/core/tx/replication/controller/controller.h> +#include <ydb/core/tx/replication/controller/controller.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/sequenceproxy/sequenceproxy.h> #include <ydb/core/tx/sequenceshard/sequenceshard.h> @@ -161,8 +161,8 @@ namespace Tests { app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()); app.SetEnableKqpSpilling(Settings->EnableKqpSpilling); app.SetKeepSnapshotTimeout(Settings->KeepSnapshotTimeout); - app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit); - app.SetChangesQueueBytesLimit(Settings->ChangesQueueBytesLimit); + app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit); + app.SetChangesQueueBytesLimit(Settings->ChangesQueueBytesLimit); app.CompactionConfig = Settings->CompactionConfig; app.FeatureFlags = Settings->FeatureFlags; @@ -300,12 +300,12 @@ namespace Tests { future.Subscribe(startCb); GRpcServer->AddService(grpcService); - GRpcServer->AddService(new NGRpcService::TGRpcYdbExportService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcYdbImportService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcYdbExportService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcYdbImportService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbSchemeService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbTableService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbScriptingService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId)); @@ -329,7 +329,7 @@ namespace Tests { } } GRpcServer->AddService(new NGRpcService::TGRpcYdbLogStoreService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcAuthService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcAuthService(system, counters, grpcRequestProxyId)); GRpcServer->Start(); } @@ -545,10 +545,10 @@ namespace Tests { TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( &NSequenceShard::CreateSequenceShard, TMailboxType::Revolving, appData.UserPoolId, TMailboxType::Revolving, appData.SystemPoolId)); - localConfig.TabletClassInfo[appData.DefaultTabletTypes.ReplicationController] = - TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( - &NReplication::CreateController, TMailboxType::Revolving, appData.UserPoolId, - TMailboxType::Revolving, appData.SystemPoolId)); + localConfig.TabletClassInfo[appData.DefaultTabletTypes.ReplicationController] = + TLocalConfig::TTabletClassInfo(new TTabletSetupInfo( + &NReplication::CreateController, TMailboxType::Revolving, appData.UserPoolId, + TMailboxType::Revolving, appData.SystemPoolId)); } void TServer::SetupLocalService(ui32 nodeIdx, const TString &domainName) { @@ -653,14 +653,14 @@ namespace Tests { } { - auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << Settings->GrpcPort); - if (!Driver) { - Driver.Reset(new NYdb::TDriver(driverConfig)); - } - Runtime->GetAppData(nodeIdx).YdbDriver = Driver.Get(); - } - - { + auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << Settings->GrpcPort); + if (!Driver) { + Driver.Reset(new NYdb::TDriver(driverConfig)); + } + Runtime->GetAppData(nodeIdx).YdbDriver = Driver.Get(); + } + + { if (Settings->PQConfig.GetEnabled() == true) { IActor *pqMetaCache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache( new NMonitoring::TDynamicCounters(), TDuration::Seconds(1) @@ -1303,23 +1303,23 @@ namespace Tests { return (NMsgBusProxy::EResponseStatus)response.GetStatus(); } - NMsgBusProxy::EResponseStatus TClient::CreateUser(const TString& parent, const TString& user, const TString& password) { - TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); - auto* op = request->Record.MutableTransaction()->MutableModifyScheme(); - op->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterLogin); - op->SetWorkingDir(parent); - - auto* createUser = op->MutableAlterLogin()->MutableCreateUser(); - createUser->SetUser(user); - createUser->SetPassword(password); - - TAutoPtr<NBus::TBusMessage> reply; - NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply); - UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); - const NKikimrClient::TResponse &response = dynamic_cast<NMsgBusProxy::TBusResponse *>(reply.Get())->Record; - return (NMsgBusProxy::EResponseStatus)response.GetStatus(); - } - + NMsgBusProxy::EResponseStatus TClient::CreateUser(const TString& parent, const TString& user, const TString& password) { + TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); + auto* op = request->Record.MutableTransaction()->MutableModifyScheme(); + op->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterLogin); + op->SetWorkingDir(parent); + + auto* createUser = op->MutableAlterLogin()->MutableCreateUser(); + createUser->SetUser(user); + createUser->SetPassword(password); + + TAutoPtr<NBus::TBusMessage> reply; + NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply); + UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); + const NKikimrClient::TResponse &response = dynamic_cast<NMsgBusProxy::TBusResponse *>(reply.Get())->Record; + return (NMsgBusProxy::EResponseStatus)response.GetStatus(); + } + NMsgBusProxy::EResponseStatus TClient::CreateTable(const TString& parent, const NKikimrSchemeOp::TTableDescription &table, TDuration timeout) { TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); @@ -1607,7 +1607,7 @@ namespace Tests { return (NMsgBusProxy::EResponseStatus)response.GetStatus(); } - TAutoPtr<NMsgBusProxy::TBusResponse> TClient::LsImpl(const TString& path) { + TAutoPtr<NMsgBusProxy::TBusResponse> TClient::LsImpl(const TString& path) { TAutoPtr<NMsgBusProxy::TBusSchemeDescribe> request(new NMsgBusProxy::TBusSchemeDescribe()); request->Record.SetPath(path); request->Record.MutableOptions()->SetShowPrivateTable(true); @@ -1620,10 +1620,10 @@ namespace Tests { return dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Release()); } - TAutoPtr<NMsgBusProxy::TBusResponse> TClient::Ls(const TString& path) { - return LsImpl(path).Release(); - } - + TAutoPtr<NMsgBusProxy::TBusResponse> TClient::Ls(const TString& path) { + return LsImpl(path).Release(); + } + TClient::TPathVersion TClient::ExtractPathVersion(const TAutoPtr<NMsgBusProxy::TBusResponse>& describe) { UNIT_ASSERT(describe.Get()); auto& record = describe->Record; @@ -1665,7 +1665,7 @@ namespace Tests { entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; runtime->Send( new IEventHandle( - MakeSchemeCacheID(), + MakeSchemeCacheID(), sender, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())), nodeIdx); diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index acbcb0d610c..2064752ab45 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -76,7 +76,7 @@ namespace Tests { NMiniKQL::IFunctionRegistry* DefaultFrFactory(const NScheme::TTypeRegistry& typeRegistry); - struct TServerSettings: public TThrRefBase, public TTestFeatureFlagsHolder<TServerSettings> { + struct TServerSettings: public TThrRefBase, public TTestFeatureFlagsHolder<TServerSettings> { static constexpr ui64 BOX_ID = 999; ui64 POOL_ID = 1; @@ -114,8 +114,8 @@ namespace Tests { bool EnableKqpSpilling = false; bool EnableYq = false; TDuration KeepSnapshotTimeout = TDuration::Zero(); - ui64 ChangesQueueItemsLimit = 0; - ui64 ChangesQueueBytesLimit = 0; + ui64 ChangesQueueItemsLimit = 0; + ui64 ChangesQueueBytesLimit = 0; NKikimrConfig::TAppConfig AppConfig; NKikimrConfig::TCompactionConfig CompactionConfig; TMap<ui32, TString> NodeKeys; @@ -156,8 +156,8 @@ namespace Tests { TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; } TServerSettings& SetEnableYq(bool value) { EnableYq = value; return *this; } TServerSettings& SetKeepSnapshotTimeout(TDuration value) { KeepSnapshotTimeout = value; return *this; } - TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } - TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; } + TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } + TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; } TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory @@ -357,7 +357,7 @@ namespace Tests { NMsgBusProxy::EResponseStatus DeleteSubdomain(const TString& parent, const TString &name); NMsgBusProxy::EResponseStatus ForceDeleteSubdomain(const TString& parent, const TString &name); NMsgBusProxy::EResponseStatus ForceDeleteUnsafe(const TString& parent, const TString &name); - NMsgBusProxy::EResponseStatus CreateUser(const TString& parent, const TString& user, const TString& password); + NMsgBusProxy::EResponseStatus CreateUser(const TString& parent, const TString& user, const TString& password); NMsgBusProxy::EResponseStatus CreateTable(const TString& parent, const TString& scheme, TDuration timeout = TDuration::Seconds(5000)); NMsgBusProxy::EResponseStatus CreateTable(const TString& parent, const NKikimrSchemeOp::TTableDescription &table, TDuration timeout = TDuration::Seconds(5000)); @@ -494,8 +494,8 @@ namespace Tests { return id - offset; } - TAutoPtr<NMsgBusProxy::TBusResponse> LsImpl(const TString& path); - + TAutoPtr<NMsgBusProxy::TBusResponse> LsImpl(const TString& path); + static void SetApplyIf(NKikimrSchemeOp::TModifyScheme& transaction, const TApplyIf& applyIf) { for (auto& pathVer: applyIf) { auto item = transaction.AddApplyIf(); diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 1d385cba325..137fa829f3a 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -73,7 +73,7 @@ PEERDIR( ydb/core/tx/coordinator ydb/core/tx/long_tx_service ydb/core/tx/mediator - ydb/core/tx/replication/controller + ydb/core/tx/replication/controller ydb/core/tx/sequenceproxy ydb/core/tx/sequenceshard ydb/core/tx/time_cast @@ -92,7 +92,7 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_driver ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs ydb/public/sdk/cpp/client/ydb_table - ydb/services/auth + ydb/services/auth ydb/services/cms ydb/services/datastreams ydb/services/discovery diff --git a/ydb/core/tx/datashard/alter_cdc_stream_unit.cpp b/ydb/core/tx/datashard/alter_cdc_stream_unit.cpp index 9425692c5a4..b79d5017361 100644 --- a/ydb/core/tx/datashard/alter_cdc_stream_unit.cpp +++ b/ydb/core/tx/datashard/alter_cdc_stream_unit.cpp @@ -1,59 +1,59 @@ -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -namespace NKikimr { +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +namespace NKikimr { namespace NDataShard { - -class TAlterCdcStreamUnit : public TExecutionUnit { -public: + +class TAlterCdcStreamUnit : public TExecutionUnit { +public: TAlterCdcStreamUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::AlterCdcStream, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { - Y_VERIFY(op->IsSchemeTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - auto& schemeTx = tx->GetSchemeTx(); - if (!schemeTx.HasAlterCdcStreamNotice()) { - return EExecutionStatus::Executed; - } - - const auto& params = schemeTx.GetAlterCdcStreamNotice(); - const auto& streamDesc = params.GetStreamDescription(); - - const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); - Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); - - const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); - Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); - + : TExecutionUnit(EExecutionUnitKind::AlterCdcStream, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { + Y_VERIFY(op->IsSchemeTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + auto& schemeTx = tx->GetSchemeTx(); + if (!schemeTx.HasAlterCdcStreamNotice()) { + return EExecutionStatus::Executed; + } + + const auto& params = schemeTx.GetAlterCdcStreamNotice(); + const auto& streamDesc = params.GetStreamDescription(); + + const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); + Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); + + const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); + Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); + Y_VERIFY_S(streamDesc.GetState() == NKikimrSchemeOp::ECdcStreamStateDisabled, "Unexpected alter cdc stream" - << ": desc# " << streamDesc.ShortDebugString()); - auto tableInfo = DataShard.AlterTableDisableCdcStream(ctx, txc, pathId, params.GetTableSchemaVersion(), streamPathId); - DataShard.AddUserTable(pathId, tableInfo); - - BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); - op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); - - return EExecutionStatus::DelayCompleteNoMoreRestarts; - } - - void Complete(TOperation::TPtr, const TActorContext&) override { - } -}; - + << ": desc# " << streamDesc.ShortDebugString()); + auto tableInfo = DataShard.AlterTableDisableCdcStream(ctx, txc, pathId, params.GetTableSchemaVersion(), streamPathId); + DataShard.AddUserTable(pathId, tableInfo); + + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); + op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); + + return EExecutionStatus::DelayCompleteNoMoreRestarts; + } + + void Complete(TOperation::TPtr, const TActorContext&) override { + } +}; + THolder<TExecutionUnit> CreateAlterCdcStreamUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TAlterCdcStreamUnit(self, pipeline)); -} - + return THolder(new TAlterCdcStreamUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/backup_restore_common.h b/ydb/core/tx/datashard/backup_restore_common.h index b1938682b91..06571ca2256 100644 --- a/ydb/core/tx/datashard/backup_restore_common.h +++ b/ydb/core/tx/datashard/backup_restore_common.h @@ -1,132 +1,132 @@ -#pragma once - -#include "defs.h" - -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit.h" - -namespace NKikimr { +#pragma once + +#include "defs.h" + +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit.h" + +namespace NKikimr { namespace NDataShard { - -template <typename TEvCancel> -class TBackupRestoreUnitBase : public TExecutionUnit { -protected: - using TBase = TBackupRestoreUnitBase<TEvCancel>; - - virtual bool IsRelevant(TActiveTransaction* tx) const = 0; - - virtual bool IsWaiting(TOperation::TPtr op) const = 0; - virtual void SetWaiting(TOperation::TPtr op) = 0; - virtual void ResetWaiting(TOperation::TPtr op) = 0; - virtual bool Run(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) = 0; - - virtual bool HasResult(TOperation::TPtr op) const = 0; - virtual void ProcessResult(TOperation::TPtr op, const TActorContext& ctx) = 0; - - virtual void Cancel(TActiveTransaction* tx, const TActorContext& ctx) = 0; - - void Abort(TOperation::TPtr op, const TActorContext& ctx, const TString& error) { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, error); - - BuildResult(op)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, error); - ResetWaiting(op); - - Cancel(tx, ctx); - } - -private: - void ProcessEvent(TAutoPtr<NActors::IEventHandle>& ev, TOperation::TPtr op, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - OHFunc(TEvCancel, Handle); - } - } - - void Handle(typename TEvCancel::TPtr&, TOperation::TPtr op, const TActorContext& ctx) { - if (IsWaiting(op)) { - Abort(op, ctx, TStringBuilder() << "Interrupted " << GetKind() << " operation " << *op - << " while waiting to finish at " << DataShard.TabletID()); - } - } - - void PersistResult(TOperation::TPtr op, TTransactionContext& txc) { - auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); - Y_VERIFY(schemeOp); - - NIceDb::TNiceDb db(txc.DB); - DataShard.PersistSchemeTxResult(db, *schemeOp); - } - -public: + +template <typename TEvCancel> +class TBackupRestoreUnitBase : public TExecutionUnit { +protected: + using TBase = TBackupRestoreUnitBase<TEvCancel>; + + virtual bool IsRelevant(TActiveTransaction* tx) const = 0; + + virtual bool IsWaiting(TOperation::TPtr op) const = 0; + virtual void SetWaiting(TOperation::TPtr op) = 0; + virtual void ResetWaiting(TOperation::TPtr op) = 0; + virtual bool Run(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) = 0; + + virtual bool HasResult(TOperation::TPtr op) const = 0; + virtual void ProcessResult(TOperation::TPtr op, const TActorContext& ctx) = 0; + + virtual void Cancel(TActiveTransaction* tx, const TActorContext& ctx) = 0; + + void Abort(TOperation::TPtr op, const TActorContext& ctx, const TString& error) { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, error); + + BuildResult(op)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, error); + ResetWaiting(op); + + Cancel(tx, ctx); + } + +private: + void ProcessEvent(TAutoPtr<NActors::IEventHandle>& ev, TOperation::TPtr op, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + OHFunc(TEvCancel, Handle); + } + } + + void Handle(typename TEvCancel::TPtr&, TOperation::TPtr op, const TActorContext& ctx) { + if (IsWaiting(op)) { + Abort(op, ctx, TStringBuilder() << "Interrupted " << GetKind() << " operation " << *op + << " while waiting to finish at " << DataShard.TabletID()); + } + } + + void PersistResult(TOperation::TPtr op, TTransactionContext& txc) { + auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); + Y_VERIFY(schemeOp); + + NIceDb::TNiceDb db(txc.DB); + DataShard.PersistSchemeTxResult(db, *schemeOp); + } + +public: TBackupRestoreUnitBase(EExecutionUnitKind kind, TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(kind, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr op) const override final { - if (!IsWaiting(op)) { - return true; - } - - if (HasResult(op)) { - return true; - } - - if (op->HasPendingInputEvents()) { - return true; - } - - return false; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override final { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - if (!IsRelevant(tx)) { - return EExecutionStatus::Executed; - } - - if (!IsWaiting(op)) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Starting a " << GetKind() << " operation" - << " at " << DataShard.TabletID()); - - if (!Run(op, txc, ctx)) { - return EExecutionStatus::Executed; - } - - SetWaiting(op); - Y_VERIFY_DEBUG(!HasResult(op)); - } - - if (HasResult(op)) { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "" << GetKind() << " complete" - << " at " << DataShard.TabletID()); - - ProcessResult(op, ctx); - PersistResult(op, txc); - ResetWaiting(op); - } - - while (op->HasPendingInputEvents()) { - ProcessEvent(op->InputEvents().front(), op, ctx); - op->InputEvents().pop(); - } - - if (IsWaiting(op)) { - return EExecutionStatus::Continue; - } - - return EExecutionStatus::Executed; - } - - void Complete(TOperation::TPtr, const TActorContext&) override final { - } - -}; // TBackupRestoreUnitBase - + : TExecutionUnit(kind, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr op) const override final { + if (!IsWaiting(op)) { + return true; + } + + if (HasResult(op)) { + return true; + } + + if (op->HasPendingInputEvents()) { + return true; + } + + return false; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override final { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + if (!IsRelevant(tx)) { + return EExecutionStatus::Executed; + } + + if (!IsWaiting(op)) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Starting a " << GetKind() << " operation" + << " at " << DataShard.TabletID()); + + if (!Run(op, txc, ctx)) { + return EExecutionStatus::Executed; + } + + SetWaiting(op); + Y_VERIFY_DEBUG(!HasResult(op)); + } + + if (HasResult(op)) { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "" << GetKind() << " complete" + << " at " << DataShard.TabletID()); + + ProcessResult(op, ctx); + PersistResult(op, txc); + ResetWaiting(op); + } + + while (op->HasPendingInputEvents()) { + ProcessEvent(op->InputEvents().front(), op, ctx); + op->InputEvents().pop(); + } + + if (IsWaiting(op)) { + return EExecutionStatus::Continue; + } + + return EExecutionStatus::Executed; + } + + void Complete(TOperation::TPtr, const TActorContext&) override final { + } + +}; // TBackupRestoreUnitBase + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/backup_unit.cpp b/ydb/core/tx/datashard/backup_unit.cpp index 7e889947813..002b3e74b2b 100644 --- a/ydb/core/tx/datashard/backup_unit.cpp +++ b/ydb/core/tx/datashard/backup_unit.cpp @@ -1,49 +1,49 @@ #include "export_iface.h" -#include "backup_restore_common.h" +#include "backup_restore_common.h" #include "execution_unit_ctors.h" -#include "export_scan.h" -#include "export_s3.h" +#include "export_scan.h" +#include "export_s3.h" namespace NKikimr { namespace NDataShard { -class TBackupUnit : public TBackupRestoreUnitBase<TEvDataShard::TEvCancelBackup> { - using IBuffer = NExportScan::IBuffer; - -protected: - bool IsRelevant(TActiveTransaction* tx) const override { - return tx->GetSchemeTx().HasBackup(); - } +class TBackupUnit : public TBackupRestoreUnitBase<TEvDataShard::TEvCancelBackup> { + using IBuffer = NExportScan::IBuffer; - bool IsWaiting(TOperation::TPtr op) const override { - return op->IsWaitingForScan(); - } +protected: + bool IsRelevant(TActiveTransaction* tx) const override { + return tx->GetSchemeTx().HasBackup(); + } - void SetWaiting(TOperation::TPtr op) override { - op->SetWaitingForScanFlag(); - } + bool IsWaiting(TOperation::TPtr op) const override { + return op->IsWaitingForScan(); + } - void ResetWaiting(TOperation::TPtr op) override { - op->ResetWaitingForScanFlag(); - } + void SetWaiting(TOperation::TPtr op) override { + op->SetWaitingForScanFlag(); + } + + void ResetWaiting(TOperation::TPtr op) override { + op->ResetWaitingForScanFlag(); + } - bool Run(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + bool Run(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - Y_VERIFY(tx->GetSchemeTx().HasBackup()); - const auto& backup = tx->GetSchemeTx().GetBackup(); + Y_VERIFY(tx->GetSchemeTx().HasBackup()); + const auto& backup = tx->GetSchemeTx().GetBackup(); - const ui64 tableId = backup.GetTableId(); + const ui64 tableId = backup.GetTableId(); Y_VERIFY(DataShard.GetUserTables().contains(tableId)); - - const ui32 localTableId = DataShard.GetUserTables().at(tableId)->LocalTid; - Y_VERIFY(txc.DB.GetScheme().GetTableInfo(localTableId)); - - auto* appData = AppData(ctx); + + const ui32 localTableId = DataShard.GetUserTables().at(tableId)->LocalTid; + Y_VERIFY(txc.DB.GetScheme().GetTableInfo(localTableId)); + + auto* appData = AppData(ctx); std::shared_ptr<::NKikimr::NDataShard::IExport> exp; - if (backup.HasYTSettings()) { + if (backup.HasYTSettings()) { auto* exportFactory = appData->DataShardExportFactory; if (exportFactory) { const auto& settings = backup.GetYTSettings(); @@ -52,7 +52,7 @@ protected: Abort(op, ctx, "Exports to YT are disabled"); return false; } - } else if (backup.HasS3Settings()) { + } else if (backup.HasS3Settings()) { auto* exportFactory = appData->DataShardExportFactory; if (exportFactory) { std::shared_ptr<IExport>(exportFactory->CreateExportToS3()).swap(exp); @@ -60,91 +60,91 @@ protected: Abort(op, ctx, "Exports to S3 are disabled"); return false; } - } else { - Abort(op, ctx, "Unsupported backup task"); - return false; - } - - const auto& columns = DataShard.GetUserTables().at(tableId)->Columns; - const auto& scanSettings = backup.GetScanSettings(); - const ui64 rowsLimit = scanSettings.GetRowsBatchSize() ? scanSettings.GetRowsBatchSize() : Max<ui64>(); - const ui64 bytesLimit = scanSettings.GetBytesBatchSize(); - - auto createUploader = [self = DataShard.SelfId(), txId = op->GetTxId(), columns, backup, exp]() { - return exp->CreateUploader(self, txId, columns, backup); - }; - + } else { + Abort(op, ctx, "Unsupported backup task"); + return false; + } + + const auto& columns = DataShard.GetUserTables().at(tableId)->Columns; + const auto& scanSettings = backup.GetScanSettings(); + const ui64 rowsLimit = scanSettings.GetRowsBatchSize() ? scanSettings.GetRowsBatchSize() : Max<ui64>(); + const ui64 bytesLimit = scanSettings.GetBytesBatchSize(); + + auto createUploader = [self = DataShard.SelfId(), txId = op->GetTxId(), columns, backup, exp]() { + return exp->CreateUploader(self, txId, columns, backup); + }; + THolder<IBuffer> buffer{exp->CreateBuffer(columns, rowsLimit, bytesLimit)}; - THolder<NTable::IScan> scan{CreateExportScan(std::move(buffer), createUploader)}; - - const auto& taskName = appData->DataShardConfig.GetBackupTaskName(); - const auto taskPrio = appData->DataShardConfig.GetBackupTaskPriority(); - + THolder<NTable::IScan> scan{CreateExportScan(std::move(buffer), createUploader)}; + + const auto& taskName = appData->DataShardConfig.GetBackupTaskName(); + const auto taskPrio = appData->DataShardConfig.GetBackupTaskPriority(); + ui64 readAheadLo = appData->DataShardConfig.GetBackupReadAheadLo(); if (ui64 readAheadLoOverride = DataShard.GetBackupReadAheadLoOverride(); readAheadLoOverride > 0) { readAheadLo = readAheadLoOverride; } - - ui64 readAheadHi = appData->DataShardConfig.GetBackupReadAheadHi(); + + ui64 readAheadHi = appData->DataShardConfig.GetBackupReadAheadHi(); if (ui64 readAheadHiOverride = DataShard.GetBackupReadAheadHiOverride(); readAheadHiOverride > 0) { readAheadHi = readAheadHiOverride; } - tx->SetScanTask(DataShard.QueueScan(localTableId, scan.Release(), op->GetTxId(), - TScanOptions() - .SetResourceBroker(taskName, taskPrio) - .SetReadAhead(readAheadLo, readAheadHi) - .SetReadPrio(TScanOptions::EReadPrio::Low) - )); + tx->SetScanTask(DataShard.QueueScan(localTableId, scan.Release(), op->GetTxId(), + TScanOptions() + .SetResourceBroker(taskName, taskPrio) + .SetReadAhead(readAheadLo, readAheadHi) + .SetReadPrio(TScanOptions::EReadPrio::Low) + )); - return true; + return true; } - bool HasResult(TOperation::TPtr op) const override { - return op->HasScanResult(); - } + bool HasResult(TOperation::TPtr op) const override { + return op->HasScanResult(); + } - void ProcessResult(TOperation::TPtr op, const TActorContext&) override { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - auto* result = CheckedCast<TExportScanProduct*>(op->ScanResult().Get()); - auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); + void ProcessResult(TOperation::TPtr op, const TActorContext&) override { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + auto* result = CheckedCast<TExportScanProduct*>(op->ScanResult().Get()); + auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); schemeOp->Success = result->Success; - schemeOp->Error = std::move(result->Error); - schemeOp->BytesProcessed = result->BytesRead; - schemeOp->RowsProcessed = result->RowsRead; + schemeOp->Error = std::move(result->Error); + schemeOp->BytesProcessed = result->BytesRead; + schemeOp->RowsProcessed = result->RowsRead; op->SetScanResult(nullptr); - tx->SetScanTask(0); + tx->SetScanTask(0); } - void Cancel(TActiveTransaction* tx, const TActorContext&) override { - if (!tx->GetScanTask()) { - return; - } + void Cancel(TActiveTransaction* tx, const TActorContext&) override { + if (!tx->GetScanTask()) { + return; + } - const ui64 tableId = tx->GetSchemeTx().GetBackup().GetTableId(); + const ui64 tableId = tx->GetSchemeTx().GetBackup().GetTableId(); Y_VERIFY(DataShard.GetUserTables().contains(tableId)); - const ui32 localTableId = DataShard.GetUserTables().at(tableId)->LocalTid; + const ui32 localTableId = DataShard.GetUserTables().at(tableId)->LocalTid; DataShard.CancelScan(localTableId, tx->GetScanTask()); tx->SetScanTask(0); } -public: +public: TBackupUnit(TDataShard& self, TPipeline& pipeline) - : TBase(EExecutionUnitKind::Backup, self, pipeline) - { - } + : TBase(EExecutionUnitKind::Backup, self, pipeline) + { + } -}; // TBackupUnit +}; // TBackupUnit THolder<TExecutionUnit> CreateBackupUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TBackupUnit(self, pipeline)); + return THolder(new TBackupUnit(self, pipeline)); } -} // NDataShard -} // NKikimr +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp b/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp index d79b7dc8c4f..ee90c2fc2b8 100644 --- a/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp +++ b/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp @@ -1,190 +1,190 @@ -#include "datashard_active_transaction.h" -#include "datashard_distributed_erase.h" -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "erase_rows_condition.h" -#include "execution_unit_ctors.h" - -#include <util/generic/bitmap.h> -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/ptr.h> - -namespace NKikimr { +#include "datashard_active_transaction.h" +#include "datashard_distributed_erase.h" +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "erase_rows_condition.h" +#include "execution_unit_ctors.h" + +#include <util/generic/bitmap.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/ptr.h> + +namespace NKikimr { namespace NDataShard { - -class TBuildDistributedEraseTxOutRSUnit : public TExecutionUnit { - static TVector<NTable::TTag> MakeTags(const TVector<NTable::TTag>& conditionTags, - const google::protobuf::RepeatedField<ui32>& indexColumnIds) { - - Y_VERIFY(conditionTags.size() == 1, "Multi-column conditions are not supported"); - TVector<NTable::TTag> tags = conditionTags; - - THashSet<NTable::TTag> uniqTags(tags.begin(), tags.end()); - for (const auto columnId : indexColumnIds) { - if (uniqTags.insert(columnId).second) { - tags.push_back(columnId); - } - } - - return tags; - } - - static TVector<TCell> ExtractIndexCells(const NTable::TRowState& row, const TVector<NTable::TTag>& tags, - const google::protobuf::RepeatedField<ui32>& indexColumnIds) { - - THashMap<ui32, ui32> tagToPos; - for (ui32 pos = 0; pos < tags.size(); ++pos) { - const auto tag = tags.at(pos); - - Y_VERIFY_DEBUG(!tagToPos.contains(tag)); - tagToPos.emplace(tag, pos); - } - - TVector<TCell> result; - for (const auto columnId : indexColumnIds) { - auto it = tagToPos.find(columnId); - Y_VERIFY(it != tagToPos.end()); - - const auto pos = it->second; - Y_VERIFY(pos < row.Size()); - - result.push_back(row.Get(pos)); - } - - return result; - } - - static bool CompareCells(const TVector<TRawTypeValue>& expectedValue, const TVector<TCell>& actualValue) { - Y_VERIFY(expectedValue.size() == actualValue.size()); - - for (ui32 pos = 0; pos < expectedValue.size(); ++pos) { - const auto& expected = expectedValue.at(pos); - const auto& actual = actualValue.at(pos); - - if (0 != CompareTypedCells(actual, expected.AsRef(), expected.Type())) { - return false; - } - } - - return true; - } - -public: + +class TBuildDistributedEraseTxOutRSUnit : public TExecutionUnit { + static TVector<NTable::TTag> MakeTags(const TVector<NTable::TTag>& conditionTags, + const google::protobuf::RepeatedField<ui32>& indexColumnIds) { + + Y_VERIFY(conditionTags.size() == 1, "Multi-column conditions are not supported"); + TVector<NTable::TTag> tags = conditionTags; + + THashSet<NTable::TTag> uniqTags(tags.begin(), tags.end()); + for (const auto columnId : indexColumnIds) { + if (uniqTags.insert(columnId).second) { + tags.push_back(columnId); + } + } + + return tags; + } + + static TVector<TCell> ExtractIndexCells(const NTable::TRowState& row, const TVector<NTable::TTag>& tags, + const google::protobuf::RepeatedField<ui32>& indexColumnIds) { + + THashMap<ui32, ui32> tagToPos; + for (ui32 pos = 0; pos < tags.size(); ++pos) { + const auto tag = tags.at(pos); + + Y_VERIFY_DEBUG(!tagToPos.contains(tag)); + tagToPos.emplace(tag, pos); + } + + TVector<TCell> result; + for (const auto columnId : indexColumnIds) { + auto it = tagToPos.find(columnId); + Y_VERIFY(it != tagToPos.end()); + + const auto pos = it->second; + Y_VERIFY(pos < row.Size()); + + result.push_back(row.Get(pos)); + } + + return result; + } + + static bool CompareCells(const TVector<TRawTypeValue>& expectedValue, const TVector<TCell>& actualValue) { + Y_VERIFY(expectedValue.size() == actualValue.size()); + + for (ui32 pos = 0; pos < expectedValue.size(); ++pos) { + const auto& expected = expectedValue.at(pos); + const auto& actual = actualValue.at(pos); + + if (0 != CompareTypedCells(actual, expected.AsRef(), expected.Type())) { + return false; + } + } + + return true; + } + +public: TBuildDistributedEraseTxOutRSUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::BuildDistributedEraseTxOutRS, true, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext&) override { - Y_VERIFY(op->IsDistributedEraseTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - const auto& eraseTx = tx->GetDistributedEraseTx(); - if (!eraseTx->HasDependents()) { - return EExecutionStatus::Executed; - } - - const auto& request = eraseTx->GetRequest(); - const ui64 tableId = request.GetTableId(); - - Y_VERIFY(DataShard.GetUserTables().contains(tableId)); - const TUserTable& tableInfo = *DataShard.GetUserTables().at(tableId); - + : TExecutionUnit(EExecutionUnitKind::BuildDistributedEraseTxOutRS, true, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext&) override { + Y_VERIFY(op->IsDistributedEraseTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + const auto& eraseTx = tx->GetDistributedEraseTx(); + if (!eraseTx->HasDependents()) { + return EExecutionStatus::Executed; + } + + const auto& request = eraseTx->GetRequest(); + const ui64 tableId = request.GetTableId(); + + Y_VERIFY(DataShard.GetUserTables().contains(tableId)); + const TUserTable& tableInfo = *DataShard.GetUserTables().at(tableId); + THolder<IEraseRowsCondition> condition{CreateEraseRowsCondition(request)}; - Y_VERIFY(condition.Get()); - condition->Prepare(txc.DB.GetRowScheme(tableInfo.LocalTid), 0); - - const auto tags = MakeTags(condition->Tags(), eraseTx->GetIndexColumnIds()); - auto readVersion = DataShard.GetReadWriteVersions(tx).ReadVersion; - bool pageFault = false; - - TDynBitMap confirmedRows; - for (ui32 i = 0; i < request.KeyColumnsSize(); ++i) { - TSerializedCellVec keyCells; - Y_VERIFY(TSerializedCellVec::TryParse(request.GetKeyColumns(i), keyCells)); - Y_VERIFY(keyCells.GetCells().size() == tableInfo.KeyColumnTypes.size()); - - TVector<TRawTypeValue> key; - for (ui32 pos = 0; pos < tableInfo.KeyColumnTypes.size(); ++pos) { - const NScheme::TTypeId type = tableInfo.KeyColumnTypes[pos]; - const TCell& cell = keyCells.GetCells()[pos]; - - key.emplace_back(TRawTypeValue(cell.AsRef(), type)); - } - - TSerializedCellVec indexCells; - TVector<TRawTypeValue> indexTypedVals; - if (!eraseTx->GetIndexColumns().empty()) { - Y_VERIFY(i < static_cast<ui32>(eraseTx->GetIndexColumns().size())); - Y_VERIFY(TSerializedCellVec::TryParse(eraseTx->GetIndexColumns().at(i), indexCells)); - Y_VERIFY(indexCells.GetCells().size() == static_cast<ui32>(eraseTx->GetIndexColumnIds().size())); - - for (ui32 pos = 0; pos < static_cast<ui32>(eraseTx->GetIndexColumnIds().size()); ++pos) { - auto it = tableInfo.Columns.find(eraseTx->GetIndexColumnIds().Get(pos)); - Y_VERIFY(it != tableInfo.Columns.end()); - - const NScheme::TTypeId type = it->second.Type; - const TCell& cell = indexCells.GetCells()[pos]; - - indexTypedVals.emplace_back(TRawTypeValue(cell.AsRef(), type)); - } - } - - NTable::TRowState row; - const auto ready = txc.DB.Select(tableInfo.LocalTid, key, tags, row, 0, readVersion); - - if (pageFault) { - continue; - } - - switch (ready) { - case NTable::EReady::Page: - pageFault = true; - break; - case NTable::EReady::Gone: - confirmedRows.Reset(i); - break; - case NTable::EReady::Data: - if (!condition->Check(row) || !CompareCells(indexTypedVals, ExtractIndexCells(row, tags, eraseTx->GetIndexColumnIds()))) { - confirmedRows.Reset(i); - } else { - confirmedRows.Set(i); - } - break; - } - } - - if (pageFault) { - return EExecutionStatus::Restart; - } - - NKikimrTxDataShard::TDistributedEraseRS rs; - rs.SetConfirmedRows(SerializeBitMap(confirmedRows)); - const TString rsBody = rs.SerializeAsString(); - - for (const auto& dependent : eraseTx->GetDependents()) { - op->OutReadSets()[std::make_pair(DataShard.TabletID(), dependent.GetShardId())] = rsBody; - } - - eraseTx->SetConfirmedRows(std::move(confirmedRows)); - - return EExecutionStatus::Executed; - } - - void Complete(TOperation::TPtr, const TActorContext&) override { - } -}; - + Y_VERIFY(condition.Get()); + condition->Prepare(txc.DB.GetRowScheme(tableInfo.LocalTid), 0); + + const auto tags = MakeTags(condition->Tags(), eraseTx->GetIndexColumnIds()); + auto readVersion = DataShard.GetReadWriteVersions(tx).ReadVersion; + bool pageFault = false; + + TDynBitMap confirmedRows; + for (ui32 i = 0; i < request.KeyColumnsSize(); ++i) { + TSerializedCellVec keyCells; + Y_VERIFY(TSerializedCellVec::TryParse(request.GetKeyColumns(i), keyCells)); + Y_VERIFY(keyCells.GetCells().size() == tableInfo.KeyColumnTypes.size()); + + TVector<TRawTypeValue> key; + for (ui32 pos = 0; pos < tableInfo.KeyColumnTypes.size(); ++pos) { + const NScheme::TTypeId type = tableInfo.KeyColumnTypes[pos]; + const TCell& cell = keyCells.GetCells()[pos]; + + key.emplace_back(TRawTypeValue(cell.AsRef(), type)); + } + + TSerializedCellVec indexCells; + TVector<TRawTypeValue> indexTypedVals; + if (!eraseTx->GetIndexColumns().empty()) { + Y_VERIFY(i < static_cast<ui32>(eraseTx->GetIndexColumns().size())); + Y_VERIFY(TSerializedCellVec::TryParse(eraseTx->GetIndexColumns().at(i), indexCells)); + Y_VERIFY(indexCells.GetCells().size() == static_cast<ui32>(eraseTx->GetIndexColumnIds().size())); + + for (ui32 pos = 0; pos < static_cast<ui32>(eraseTx->GetIndexColumnIds().size()); ++pos) { + auto it = tableInfo.Columns.find(eraseTx->GetIndexColumnIds().Get(pos)); + Y_VERIFY(it != tableInfo.Columns.end()); + + const NScheme::TTypeId type = it->second.Type; + const TCell& cell = indexCells.GetCells()[pos]; + + indexTypedVals.emplace_back(TRawTypeValue(cell.AsRef(), type)); + } + } + + NTable::TRowState row; + const auto ready = txc.DB.Select(tableInfo.LocalTid, key, tags, row, 0, readVersion); + + if (pageFault) { + continue; + } + + switch (ready) { + case NTable::EReady::Page: + pageFault = true; + break; + case NTable::EReady::Gone: + confirmedRows.Reset(i); + break; + case NTable::EReady::Data: + if (!condition->Check(row) || !CompareCells(indexTypedVals, ExtractIndexCells(row, tags, eraseTx->GetIndexColumnIds()))) { + confirmedRows.Reset(i); + } else { + confirmedRows.Set(i); + } + break; + } + } + + if (pageFault) { + return EExecutionStatus::Restart; + } + + NKikimrTxDataShard::TDistributedEraseRS rs; + rs.SetConfirmedRows(SerializeBitMap(confirmedRows)); + const TString rsBody = rs.SerializeAsString(); + + for (const auto& dependent : eraseTx->GetDependents()) { + op->OutReadSets()[std::make_pair(DataShard.TabletID(), dependent.GetShardId())] = rsBody; + } + + eraseTx->SetConfirmedRows(std::move(confirmedRows)); + + return EExecutionStatus::Executed; + } + + void Complete(TOperation::TPtr, const TActorContext&) override { + } +}; + THolder<TExecutionUnit> CreateBuildDistributedEraseTxOutRSUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TBuildDistributedEraseTxOutRSUnit(self, pipeline)); -} - + return THolder(new TBuildDistributedEraseTxOutRSUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/change_collector.cpp b/ydb/core/tx/datashard/change_collector.cpp index be0b6f664b3..be21b22cbf7 100644 --- a/ydb/core/tx/datashard/change_collector.cpp +++ b/ydb/core/tx/datashard/change_collector.cpp @@ -1,110 +1,110 @@ -#include "change_collector.h" -#include "change_collector_async_index.h" -#include "change_collector_cdc_stream.h" -#include "datashard_impl.h" - -#include <util/generic/vector.h> - -namespace NKikimr { +#include "change_collector.h" +#include "change_collector_async_index.h" +#include "change_collector_cdc_stream.h" +#include "datashard_impl.h" + +#include <util/generic/vector.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NMiniKQL; - -class TChangeCollectorProxy: public IChangeCollector { -public: - void AddUnderlying(THolder<IChangeCollector> collector) { - Underlying.emplace_back(std::move(collector)); - } - - bool NeedToReadKeys() const override { - for (const auto& collector : Underlying) { - if (collector->NeedToReadKeys()) { - return true; - } - } - - return false; - } - - void SetReadVersion(const TRowVersion& readVersion) override { - for (auto& collector : Underlying) { - collector->SetReadVersion(readVersion); - } - } - - void SetWriteVersion(const TRowVersion& writeVersion) override { - for (auto& collector : Underlying) { - collector->SetWriteVersion(writeVersion); - } - } - + +using namespace NMiniKQL; + +class TChangeCollectorProxy: public IChangeCollector { +public: + void AddUnderlying(THolder<IChangeCollector> collector) { + Underlying.emplace_back(std::move(collector)); + } + + bool NeedToReadKeys() const override { + for (const auto& collector : Underlying) { + if (collector->NeedToReadKeys()) { + return true; + } + } + + return false; + } + + void SetReadVersion(const TRowVersion& readVersion) override { + for (auto& collector : Underlying) { + collector->SetReadVersion(readVersion); + } + } + + void SetWriteVersion(const TRowVersion& writeVersion) override { + for (auto& collector : Underlying) { + collector->SetWriteVersion(writeVersion); + } + } + bool Collect(const TTableId& tableId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override - { - for (auto& collector : Underlying) { - if (!collector->Collect(tableId, rop, key, updates)) { - return false; - } - } - - return true; - } - - const TVector<TChange>& GetCollected() const override { - CollectedBuf.clear(); - - for (const auto& collector : Underlying) { - const auto& collected = collector->GetCollected(); - std::copy(collected.begin(), collected.end(), std::back_inserter(CollectedBuf)); - } - - return CollectedBuf; - } - - TVector<TChange>&& GetCollected() override { - CollectedBuf.clear(); - - for (auto& collector : Underlying) { - auto collected = std::move(collector->GetCollected()); - std::move(collected.begin(), collected.end(), std::back_inserter(CollectedBuf)); - } - - return std::move(CollectedBuf); - } - - void Reset() override { - for (auto& collector : Underlying) { - collector->Reset(); - } - - CollectedBuf.clear(); - } - -private: - TVector<THolder<IChangeCollector>> Underlying; - mutable TVector<TChange> CollectedBuf; - -}; // TChangeCollectorProxy - -IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, const TUserTable& table, bool isImmediateTx) { - auto proxy = MakeHolder<TChangeCollectorProxy>(); - - if (table.HasAsyncIndexes()) { - proxy->AddUnderlying(MakeHolder<TAsyncIndexChangeCollector>(&dataShard, db, isImmediateTx)); - } - - if (table.HasCdcStreams()) { - proxy->AddUnderlying(MakeHolder<TCdcStreamChangeCollector>(&dataShard, db, isImmediateTx)); - } - - return proxy.Release(); -} - -IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, ui64 tableId, bool isImmediateTx) { - Y_VERIFY(dataShard.GetUserTables().contains(tableId)); - const TUserTable& tableInfo = *dataShard.GetUserTables().at(tableId); - return CreateChangeCollector(dataShard, db, tableInfo, isImmediateTx); -} - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override + { + for (auto& collector : Underlying) { + if (!collector->Collect(tableId, rop, key, updates)) { + return false; + } + } + + return true; + } + + const TVector<TChange>& GetCollected() const override { + CollectedBuf.clear(); + + for (const auto& collector : Underlying) { + const auto& collected = collector->GetCollected(); + std::copy(collected.begin(), collected.end(), std::back_inserter(CollectedBuf)); + } + + return CollectedBuf; + } + + TVector<TChange>&& GetCollected() override { + CollectedBuf.clear(); + + for (auto& collector : Underlying) { + auto collected = std::move(collector->GetCollected()); + std::move(collected.begin(), collected.end(), std::back_inserter(CollectedBuf)); + } + + return std::move(CollectedBuf); + } + + void Reset() override { + for (auto& collector : Underlying) { + collector->Reset(); + } + + CollectedBuf.clear(); + } + +private: + TVector<THolder<IChangeCollector>> Underlying; + mutable TVector<TChange> CollectedBuf; + +}; // TChangeCollectorProxy + +IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, const TUserTable& table, bool isImmediateTx) { + auto proxy = MakeHolder<TChangeCollectorProxy>(); + + if (table.HasAsyncIndexes()) { + proxy->AddUnderlying(MakeHolder<TAsyncIndexChangeCollector>(&dataShard, db, isImmediateTx)); + } + + if (table.HasCdcStreams()) { + proxy->AddUnderlying(MakeHolder<TCdcStreamChangeCollector>(&dataShard, db, isImmediateTx)); + } + + return proxy.Release(); +} + +IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, ui64 tableId, bool isImmediateTx) { + Y_VERIFY(dataShard.GetUserTables().contains(tableId)); + const TUserTable& tableInfo = *dataShard.GetUserTables().at(tableId); + return CreateChangeCollector(dataShard, db, tableInfo, isImmediateTx); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector.h b/ydb/core/tx/datashard/change_collector.h index 29176dae895..7b24a8d46bd 100644 --- a/ydb/core/tx/datashard/change_collector.h +++ b/ydb/core/tx/datashard/change_collector.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <ydb/core/engine/minikql/change_collector_iface.h> #include <ydb/core/tablet_flat/flat_database.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -class TDataShard; -struct TUserTable; - -NMiniKQL::IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, const TUserTable& table, bool isImmediateTx); -NMiniKQL::IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, ui64 tableId, bool isImmediateTx); - + +class TDataShard; +struct TUserTable; + +NMiniKQL::IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, const TUserTable& table, bool isImmediateTx); +NMiniKQL::IChangeCollector* CreateChangeCollector(TDataShard& dataShard, NTable::TDatabase& db, ui64 tableId, bool isImmediateTx); + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_async_index.cpp b/ydb/core/tx/datashard/change_collector_async_index.cpp index 09fd3aff186..6d6c068b11e 100644 --- a/ydb/core/tx/datashard/change_collector_async_index.cpp +++ b/ydb/core/tx/datashard/change_collector_async_index.cpp @@ -1,302 +1,302 @@ -#include "change_collector_async_index.h" -#include "datashard_impl.h" - +#include "change_collector_async_index.h" +#include "datashard_impl.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -using namespace NMiniKQL; -using namespace NTable; - -class TCachedTagsBuilder { - using TCachedTags = TAsyncIndexChangeCollector::TCachedTags; - -public: - void MakeKeyTagToPos(const TVector<TTag>& keyTags) { - for (TPos pos = 0; pos < keyTags.size(); ++pos) { - KeyTagToPos.emplace(keyTags.at(pos), pos); - } - } - - void AddIndexTags(const TVector<TTag>& tags) { - IndexTags.insert(tags.begin(), tags.end()); - } - - void AddDataTags(const TVector<TTag>& tags) { - DataTags.insert(tags.begin(), tags.end()); - } - - TCachedTags Build() { - TVector<TTag> tags(Reserve(IndexTags.size() + DataTags.size())); - - for (const auto tag : IndexTags) { - tags.push_back(tag); - } - - for (const auto tag : DataTags) { - tags.push_back(tag); - } - - Y_VERIFY(!tags.empty()); - Y_VERIFY(!IndexTags.empty()); - Y_VERIFY(!KeyTagToPos.empty()); - - return TCachedTags(std::move(KeyTagToPos), std::move(tags), std::make_pair(0, IndexTags.size() - 1)); - } - -private: - THashMap<TTag, TPos> KeyTagToPos; - THashSet<TTag> IndexTags; - THashSet<TTag> DataTags; -}; - -template <typename C, typename E> -static THashMap<TTag, TPos> MakeTagToPos(const C& container, E extractor) { - THashMap<TTag, TPos> tagToPos; - - for (ui32 i = 0; i < container.size(); ++i) { - const auto tag = extractor(container.at(i)); - Y_VERIFY_DEBUG(!tagToPos.contains(tag)); - tagToPos.emplace(tag, i); - } - - return tagToPos; -} - -bool TAsyncIndexChangeCollector::NeedToReadKeys() const { - return true; -} - -void TAsyncIndexChangeCollector::SetReadVersion(const TRowVersion& readVersion) { - ReadVersion = readVersion; -} - + +using namespace NMiniKQL; +using namespace NTable; + +class TCachedTagsBuilder { + using TCachedTags = TAsyncIndexChangeCollector::TCachedTags; + +public: + void MakeKeyTagToPos(const TVector<TTag>& keyTags) { + for (TPos pos = 0; pos < keyTags.size(); ++pos) { + KeyTagToPos.emplace(keyTags.at(pos), pos); + } + } + + void AddIndexTags(const TVector<TTag>& tags) { + IndexTags.insert(tags.begin(), tags.end()); + } + + void AddDataTags(const TVector<TTag>& tags) { + DataTags.insert(tags.begin(), tags.end()); + } + + TCachedTags Build() { + TVector<TTag> tags(Reserve(IndexTags.size() + DataTags.size())); + + for (const auto tag : IndexTags) { + tags.push_back(tag); + } + + for (const auto tag : DataTags) { + tags.push_back(tag); + } + + Y_VERIFY(!tags.empty()); + Y_VERIFY(!IndexTags.empty()); + Y_VERIFY(!KeyTagToPos.empty()); + + return TCachedTags(std::move(KeyTagToPos), std::move(tags), std::make_pair(0, IndexTags.size() - 1)); + } + +private: + THashMap<TTag, TPos> KeyTagToPos; + THashSet<TTag> IndexTags; + THashSet<TTag> DataTags; +}; + +template <typename C, typename E> +static THashMap<TTag, TPos> MakeTagToPos(const C& container, E extractor) { + THashMap<TTag, TPos> tagToPos; + + for (ui32 i = 0; i < container.size(); ++i) { + const auto tag = extractor(container.at(i)); + Y_VERIFY_DEBUG(!tagToPos.contains(tag)); + tagToPos.emplace(tag, i); + } + + return tagToPos; +} + +bool TAsyncIndexChangeCollector::NeedToReadKeys() const { + return true; +} + +void TAsyncIndexChangeCollector::SetReadVersion(const TRowVersion& readVersion) { + ReadVersion = readVersion; +} + bool TAsyncIndexChangeCollector::Collect(const TTableId& tableId, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) -{ - Y_VERIFY_S(Self->IsUserTable(tableId), "Unknown table: " << tableId); - const auto localTableId = Self->GetLocalTableId(tableId); - - auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); - Y_VERIFY_S(key.size() == userTable->KeyColumnIds.size(), "Count doesn't match" - << ": key# " << key.size() - << ", tags# " << userTable->KeyColumnIds.size()); - - switch (rop) { + TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) +{ + Y_VERIFY_S(Self->IsUserTable(tableId), "Unknown table: " << tableId); + const auto localTableId = Self->GetLocalTableId(tableId); + + auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); + Y_VERIFY_S(key.size() == userTable->KeyColumnIds.size(), "Count doesn't match" + << ": key# " << key.size() + << ", tags# " << userTable->KeyColumnIds.size()); + + switch (rop) { case ERowOp::Upsert: case ERowOp::Erase: case ERowOp::Reset: - break; - default: - Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); - } - - const auto& keyTagToPos = GetKeyTagToPos(tableId); - const auto tagsToSelect = GetTagsToSelect(tableId, rop); - - TRowState row; - const auto ready = RowsCache.SelectRow(Db, localTableId, key, keyTagToPos, tagsToSelect, row, ReadVersion); - - if (ready == EReady::Page) { - return false; - } - - const bool generateDeletions = (ready != EReady::Gone); + break; + default: + Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); + } + + const auto& keyTagToPos = GetKeyTagToPos(tableId); + const auto tagsToSelect = GetTagsToSelect(tableId, rop); + + TRowState row; + const auto ready = RowsCache.SelectRow(Db, localTableId, key, keyTagToPos, tagsToSelect, row, ReadVersion); + + if (ready == EReady::Page) { + return false; + } + + const bool generateDeletions = (ready != EReady::Gone); const bool generateUpdates = (rop != ERowOp::Erase); - - if (!generateDeletions && !generateUpdates) { - return true; - } - - const auto tagToPos = MakeTagToPos(tagsToSelect, [](const auto tag) { return tag; }); - const auto updatedTagToPos = MakeTagToPos(updates, [](const TUpdateOp& op) { return op.Tag; }); - - for (const auto& [pathId, index] : userTable->Indexes) { - if (index.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { - continue; - } - - if (generateDeletions) { - bool needDeletion = false; - - for (const auto tag : index.KeyColumnIds) { + + if (!generateDeletions && !generateUpdates) { + return true; + } + + const auto tagToPos = MakeTagToPos(tagsToSelect, [](const auto tag) { return tag; }); + const auto updatedTagToPos = MakeTagToPos(updates, [](const TUpdateOp& op) { return op.Tag; }); + + for (const auto& [pathId, index] : userTable->Indexes) { + if (index.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { + continue; + } + + if (generateDeletions) { + bool needDeletion = false; + + for (const auto tag : index.KeyColumnIds) { if (updatedTagToPos.contains(tag) || rop == ERowOp::Erase || rop == ERowOp::Reset) { - needDeletion = true; - } - - Y_VERIFY(tagToPos.contains(tag)); - Y_VERIFY(userTable->Columns.contains(tag)); - FillKeyFromRowState(tag, tagToPos.at(tag), row, userTable->Columns.at(tag).Type); - } - - for (TPos pos = 0; pos < userTable->KeyColumnIds.size(); ++pos) { - FillKeyFromKey(userTable->KeyColumnIds.at(pos), pos, key); - } - - if (needDeletion) { + needDeletion = true; + } + + Y_VERIFY(tagToPos.contains(tag)); + Y_VERIFY(userTable->Columns.contains(tag)); + FillKeyFromRowState(tag, tagToPos.at(tag), row, userTable->Columns.at(tag).Type); + } + + for (TPos pos = 0; pos < userTable->KeyColumnIds.size(); ++pos) { + FillKeyFromKey(userTable->KeyColumnIds.at(pos), pos, key); + } + + if (needDeletion) { Persist(pathId, ERowOp::Erase, IndexKeyVals, IndexKeyTags, {}); - } - - Clear(); - } - - if (generateUpdates) { - bool needUpdate = false; - - for (const auto tag : index.KeyColumnIds) { - if (updatedTagToPos.contains(tag)) { - needUpdate = true; - FillKeyFromUpdate(tag, updatedTagToPos.at(tag), updates); - } else { - Y_VERIFY(userTable->Columns.contains(tag)); - const auto& column = userTable->Columns.at(tag); - + } + + Clear(); + } + + if (generateUpdates) { + bool needUpdate = false; + + for (const auto tag : index.KeyColumnIds) { + if (updatedTagToPos.contains(tag)) { + needUpdate = true; + FillKeyFromUpdate(tag, updatedTagToPos.at(tag), updates); + } else { + Y_VERIFY(userTable->Columns.contains(tag)); + const auto& column = userTable->Columns.at(tag); + if (rop == ERowOp::Reset && !column.IsKey) { - needUpdate = true; - FillKeyWithNull(tag, column.Type); - } else { - Y_VERIFY(tagToPos.contains(tag)); - FillKeyFromRowState(tag, tagToPos.at(tag), row, column.Type); - } - } - } - - for (TPos pos = 0; pos < userTable->KeyColumnIds.size(); ++pos) { - FillKeyFromKey(userTable->KeyColumnIds.at(pos), pos, key); - } - - for (const auto tag : index.DataColumnIds) { - if (updatedTagToPos.contains(tag)) { - needUpdate = true; - FillDataFromUpdate(tag, updatedTagToPos.at(tag), updates); + needUpdate = true; + FillKeyWithNull(tag, column.Type); + } else { + Y_VERIFY(tagToPos.contains(tag)); + FillKeyFromRowState(tag, tagToPos.at(tag), row, column.Type); + } + } + } + + for (TPos pos = 0; pos < userTable->KeyColumnIds.size(); ++pos) { + FillKeyFromKey(userTable->KeyColumnIds.at(pos), pos, key); + } + + for (const auto tag : index.DataColumnIds) { + if (updatedTagToPos.contains(tag)) { + needUpdate = true; + FillDataFromUpdate(tag, updatedTagToPos.at(tag), updates); } else if (rop == ERowOp::Reset) { - needUpdate = true; - Y_VERIFY(userTable->Columns.contains(tag)); - FillDataWithNull(tag, userTable->Columns.at(tag).Type); - } - } - - if (needUpdate) { + needUpdate = true; + Y_VERIFY(userTable->Columns.contains(tag)); + FillDataWithNull(tag, userTable->Columns.at(tag).Type); + } + } + + if (needUpdate) { Persist(pathId, ERowOp::Upsert, IndexKeyVals, IndexKeyTags, IndexDataVals); - } - - Clear(); - } - } - - RowsCache.UpdateCachedRow(localTableId, rop, key, updates); - return true; -} - -auto TAsyncIndexChangeCollector::CacheTags(const TTableId& tableId) const { - Y_VERIFY(Self->GetUserTables().contains(tableId.PathId.LocalPathId)); - auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); - - TCachedTagsBuilder builder; - builder.MakeKeyTagToPos(userTable->KeyColumnIds); - - for (const auto& [_, index] : userTable->Indexes) { - if (index.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { - continue; - } - - builder.AddIndexTags(index.KeyColumnIds); - builder.AddDataTags(index.DataColumnIds); - } - - return CachedTags.emplace(tableId, builder.Build()).first; -} - -const THashMap<TTag, TPos>& TAsyncIndexChangeCollector::GetKeyTagToPos(const TTableId& tableId) const { - auto it = CachedTags.find(tableId); - if (it == CachedTags.end()) { - it = CacheTags(tableId); - } - - return it->second.KeyTagToPos; -} - + } + + Clear(); + } + } + + RowsCache.UpdateCachedRow(localTableId, rop, key, updates); + return true; +} + +auto TAsyncIndexChangeCollector::CacheTags(const TTableId& tableId) const { + Y_VERIFY(Self->GetUserTables().contains(tableId.PathId.LocalPathId)); + auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); + + TCachedTagsBuilder builder; + builder.MakeKeyTagToPos(userTable->KeyColumnIds); + + for (const auto& [_, index] : userTable->Indexes) { + if (index.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { + continue; + } + + builder.AddIndexTags(index.KeyColumnIds); + builder.AddDataTags(index.DataColumnIds); + } + + return CachedTags.emplace(tableId, builder.Build()).first; +} + +const THashMap<TTag, TPos>& TAsyncIndexChangeCollector::GetKeyTagToPos(const TTableId& tableId) const { + auto it = CachedTags.find(tableId); + if (it == CachedTags.end()) { + it = CacheTags(tableId); + } + + return it->second.KeyTagToPos; +} + TArrayRef<TTag> TAsyncIndexChangeCollector::GetTagsToSelect(const TTableId& tableId, ERowOp rop) const { - auto it = CachedTags.find(tableId); - if (it == CachedTags.end()) { - it = CacheTags(tableId); - } - - switch (rop) { + auto it = CachedTags.find(tableId); + if (it == CachedTags.end()) { + it = CacheTags(tableId); + } + + switch (rop) { case ERowOp::Upsert: - return it->second.Columns; + return it->second.Columns; case ERowOp::Erase: case ERowOp::Reset: - return it->second.IndexColumns; - default: - Y_FAIL("unreachable"); - } -} - -void TAsyncIndexChangeCollector::FillKeyFromRowState(TTag tag, TPos pos, const TRowState& rowState, NScheme::TTypeId type) { - Y_VERIFY(pos < rowState.Size()); - - IndexKeyVals.emplace_back(rowState.Get(pos).AsRef(), type); - IndexKeyTags.emplace_back(tag); - TagsSeen.insert(tag); -} - -void TAsyncIndexChangeCollector::FillKeyFromKey(TTag tag, TPos pos, TArrayRef<const TRawTypeValue> key) { - Y_VERIFY(pos < key.size()); - - if (TagsSeen.contains(tag)) { - return; - } - - IndexKeyVals.emplace_back(key.at(pos)); - IndexKeyTags.emplace_back(tag); -} - -void TAsyncIndexChangeCollector::FillKeyFromUpdate(TTag tag, TPos pos, TArrayRef<const TUpdateOp> updates) { - Y_VERIFY(pos < updates.size()); - - const auto& update = updates.at(pos); + return it->second.IndexColumns; + default: + Y_FAIL("unreachable"); + } +} + +void TAsyncIndexChangeCollector::FillKeyFromRowState(TTag tag, TPos pos, const TRowState& rowState, NScheme::TTypeId type) { + Y_VERIFY(pos < rowState.Size()); + + IndexKeyVals.emplace_back(rowState.Get(pos).AsRef(), type); + IndexKeyTags.emplace_back(tag); + TagsSeen.insert(tag); +} + +void TAsyncIndexChangeCollector::FillKeyFromKey(TTag tag, TPos pos, TArrayRef<const TRawTypeValue> key) { + Y_VERIFY(pos < key.size()); + + if (TagsSeen.contains(tag)) { + return; + } + + IndexKeyVals.emplace_back(key.at(pos)); + IndexKeyTags.emplace_back(tag); +} + +void TAsyncIndexChangeCollector::FillKeyFromUpdate(TTag tag, TPos pos, TArrayRef<const TUpdateOp> updates) { + Y_VERIFY(pos < updates.size()); + + const auto& update = updates.at(pos); Y_VERIFY_S(update.Op == ECellOp::Set, "Unexpected op: " << update.Op.Raw()); - - IndexKeyVals.emplace_back(update.Value); - IndexKeyTags.emplace_back(tag); - TagsSeen.insert(tag); -} - -void TAsyncIndexChangeCollector::FillKeyWithNull(TTag tag, NScheme::TTypeId type) { - IndexKeyVals.emplace_back(TRawTypeValue({}, type)); - IndexKeyTags.emplace_back(tag); - TagsSeen.insert(tag); -} - -void TAsyncIndexChangeCollector::FillDataFromUpdate(TTag tag, TPos pos, TArrayRef<const TUpdateOp> updates) { - Y_VERIFY(pos < updates.size()); - - const auto& update = updates.at(pos); + + IndexKeyVals.emplace_back(update.Value); + IndexKeyTags.emplace_back(tag); + TagsSeen.insert(tag); +} + +void TAsyncIndexChangeCollector::FillKeyWithNull(TTag tag, NScheme::TTypeId type) { + IndexKeyVals.emplace_back(TRawTypeValue({}, type)); + IndexKeyTags.emplace_back(tag); + TagsSeen.insert(tag); +} + +void TAsyncIndexChangeCollector::FillDataFromUpdate(TTag tag, TPos pos, TArrayRef<const TUpdateOp> updates) { + Y_VERIFY(pos < updates.size()); + + const auto& update = updates.at(pos); Y_VERIFY_S(update.Op == ECellOp::Set, "Unexpected op: " << update.Op.Raw()); - + IndexDataVals.emplace_back(tag, ECellOp::Set, update.Value); -} - -void TAsyncIndexChangeCollector::FillDataWithNull(TTag tag, NScheme::TTypeId type) { +} + +void TAsyncIndexChangeCollector::FillDataWithNull(TTag tag, NScheme::TTypeId type) { IndexDataVals.emplace_back(tag, ECellOp::Set, TRawTypeValue({}, type)); -} - +} + void TAsyncIndexChangeCollector::Persist(const TPathId& pathId, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) -{ - NKikimrChangeExchange::TChangeRecord::TDataChange body; - Serialize(body, rop, key, keyTags, updates); - TBaseChangeCollector::Persist(TChangeRecord::EKind::AsyncIndex, pathId, body); -} - -void TAsyncIndexChangeCollector::Clear() { - TagsSeen.clear(); - IndexKeyTags.clear(); - IndexKeyVals.clear(); - IndexDataVals.clear(); -} - + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) +{ + NKikimrChangeExchange::TChangeRecord::TDataChange body; + Serialize(body, rop, key, keyTags, updates); + TBaseChangeCollector::Persist(TChangeRecord::EKind::AsyncIndex, pathId, body); +} + +void TAsyncIndexChangeCollector::Clear() { + TagsSeen.clear(); + IndexKeyTags.clear(); + IndexKeyVals.clear(); + IndexDataVals.clear(); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_async_index.h b/ydb/core/tx/datashard/change_collector_async_index.h index 19349a9fb76..d908dc7ef2a 100644 --- a/ydb/core/tx/datashard/change_collector_async_index.h +++ b/ydb/core/tx/datashard/change_collector_async_index.h @@ -1,75 +1,75 @@ -#pragma once - -#include "change_collector_base.h" -#include "change_collector_helpers.h" - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/vector.h> - -namespace NKikimr { +#pragma once + +#include "change_collector_base.h" +#include "change_collector_helpers.h" + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/vector.h> + +namespace NKikimr { namespace NDataShard { - -class TCachedTagsBuilder; - -class TAsyncIndexChangeCollector: public TBaseChangeCollector { - friend class TCachedTagsBuilder; - - struct TCachedTags { - explicit TCachedTags( - THashMap<NTable::TTag, NTable::TPos>&& keyTagToPos, - TVector<NTable::TTag>&& columns, - const std::pair<ui32, ui32>& indexRange) - : KeyTagToPos(std::move(keyTagToPos)) - , Columns(std::move(columns)) - , IndexColumns(&Columns.at(indexRange.first), indexRange.second + 1) - { - } - - THashMap<NTable::TTag, NTable::TPos> KeyTagToPos; - TVector<NTable::TTag> Columns; // Index + Data - TArrayRef<NTable::TTag> IndexColumns; - }; - - auto CacheTags(const TTableId& tableId) const; - const THashMap<NTable::TTag, NTable::TPos>& GetKeyTagToPos(const TTableId& tableId) const; + +class TCachedTagsBuilder; + +class TAsyncIndexChangeCollector: public TBaseChangeCollector { + friend class TCachedTagsBuilder; + + struct TCachedTags { + explicit TCachedTags( + THashMap<NTable::TTag, NTable::TPos>&& keyTagToPos, + TVector<NTable::TTag>&& columns, + const std::pair<ui32, ui32>& indexRange) + : KeyTagToPos(std::move(keyTagToPos)) + , Columns(std::move(columns)) + , IndexColumns(&Columns.at(indexRange.first), indexRange.second + 1) + { + } + + THashMap<NTable::TTag, NTable::TPos> KeyTagToPos; + TVector<NTable::TTag> Columns; // Index + Data + TArrayRef<NTable::TTag> IndexColumns; + }; + + auto CacheTags(const TTableId& tableId) const; + const THashMap<NTable::TTag, NTable::TPos>& GetKeyTagToPos(const TTableId& tableId) const; TArrayRef<NTable::TTag> GetTagsToSelect(const TTableId& tableId, NTable::ERowOp rop) const; - - void FillKeyFromRowState(NTable::TTag tag, NTable::TPos pos, const NTable::TRowState& rowState, NScheme::TTypeId type); - void FillKeyFromKey(NTable::TTag tag, NTable::TPos pos, TArrayRef<const TRawTypeValue> key); - void FillKeyFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); - void FillKeyWithNull(NTable::TTag tag, NScheme::TTypeId type); - void FillDataFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); - void FillDataWithNull(NTable::TTag tag, NScheme::TTypeId type); - + + void FillKeyFromRowState(NTable::TTag tag, NTable::TPos pos, const NTable::TRowState& rowState, NScheme::TTypeId type); + void FillKeyFromKey(NTable::TTag tag, NTable::TPos pos, TArrayRef<const TRawTypeValue> key); + void FillKeyFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); + void FillKeyWithNull(NTable::TTag tag, NScheme::TTypeId type); + void FillDataFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); + void FillDataWithNull(NTable::TTag tag, NScheme::TTypeId type); + void Persist(const TPathId& pathId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, - TArrayRef<const NTable::TUpdateOp> updates); - - void Clear(); - -public: - using TBaseChangeCollector::TBaseChangeCollector; - - bool NeedToReadKeys() const override; - void SetReadVersion(const TRowVersion& readVersion) override; - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, + TArrayRef<const NTable::TUpdateOp> updates); + + void Clear(); + +public: + using TBaseChangeCollector::TBaseChangeCollector; + + bool NeedToReadKeys() const override; + void SetReadVersion(const TRowVersion& readVersion) override; + bool Collect(const TTableId& tableId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override; - -private: - TRowVersion ReadVersion; - - mutable THashMap<TTableId, TCachedTags> CachedTags; - TRowsCache RowsCache; - - // reused between Collect() calls, cleared after every Clear() call - THashSet<NTable::TTag> TagsSeen; - TVector<NTable::TTag> IndexKeyTags; - TVector<TRawTypeValue> IndexKeyVals; - TVector<NTable::TUpdateOp> IndexDataVals; - -}; // TAsyncIndexChangeCollector - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override; + +private: + TRowVersion ReadVersion; + + mutable THashMap<TTableId, TCachedTags> CachedTags; + TRowsCache RowsCache; + + // reused between Collect() calls, cleared after every Clear() call + THashSet<NTable::TTag> TagsSeen; + TVector<NTable::TTag> IndexKeyTags; + TVector<TRawTypeValue> IndexKeyVals; + TVector<NTable::TUpdateOp> IndexDataVals; + +}; // TAsyncIndexChangeCollector + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_base.cpp b/ydb/core/tx/datashard/change_collector_base.cpp index a166ecf71be..1df9173f1f4 100644 --- a/ydb/core/tx/datashard/change_collector_base.cpp +++ b/ydb/core/tx/datashard/change_collector_base.cpp @@ -1,146 +1,146 @@ -#include "change_collector_base.h" -#include "datashard_impl.h" - +#include "change_collector_base.h" +#include "datashard_impl.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/util/yverify_stream.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -using namespace NMiniKQL; -using namespace NTable; - + +using namespace NMiniKQL; +using namespace NTable; + TBaseChangeCollector::TBaseChangeCollector(TDataShard* self, TDatabase& db, bool isImmediateTx) - : Self(self) - , Db(db) -{ - if (!isImmediateTx) { - Group = 0; - } -} - -bool TBaseChangeCollector::NeedToReadKeys() const { - return false; -} - -void TBaseChangeCollector::SetReadVersion(const TRowVersion& readVersion) { - Y_UNUSED(readVersion); -} - -void TBaseChangeCollector::SetWriteVersion(const TRowVersion& writeVersion) { - WriteVersion = writeVersion; -} - -const TVector<IChangeCollector::TChange>& TBaseChangeCollector::GetCollected() const { - return Collected; -} - -TVector<IChangeCollector::TChange>&& TBaseChangeCollector::GetCollected() { - return std::move(Collected); -} - -void TBaseChangeCollector::Reset() { - Collected.clear(); -} - -void TBaseChangeCollector::SerializeCells(TSerializedCells& out, TArrayRef<const TRawTypeValue> in, TArrayRef<const TTag> tags) { - Y_VERIFY_S(in.size() == tags.size(), "Count doesn't match" - << ": in# " << in.size() - << ", tags# " << tags.size()); - - TVector<TCell> cells(Reserve(in.size())); - for (size_t i = 0; i < in.size(); ++i) { - out.AddTags(tags.at(i)); - cells.emplace_back(in.at(i).AsRef()); - } - - out.SetData(TSerializedCellVec::Serialize(cells)); -} - -void TBaseChangeCollector::SerializeCells(TSerializedCells& out, TArrayRef<const TUpdateOp> in) { - if (!in) { - return; - } - - TVector<TCell> cells(Reserve(in.size())); - for (const auto& op : in) { + : Self(self) + , Db(db) +{ + if (!isImmediateTx) { + Group = 0; + } +} + +bool TBaseChangeCollector::NeedToReadKeys() const { + return false; +} + +void TBaseChangeCollector::SetReadVersion(const TRowVersion& readVersion) { + Y_UNUSED(readVersion); +} + +void TBaseChangeCollector::SetWriteVersion(const TRowVersion& writeVersion) { + WriteVersion = writeVersion; +} + +const TVector<IChangeCollector::TChange>& TBaseChangeCollector::GetCollected() const { + return Collected; +} + +TVector<IChangeCollector::TChange>&& TBaseChangeCollector::GetCollected() { + return std::move(Collected); +} + +void TBaseChangeCollector::Reset() { + Collected.clear(); +} + +void TBaseChangeCollector::SerializeCells(TSerializedCells& out, TArrayRef<const TRawTypeValue> in, TArrayRef<const TTag> tags) { + Y_VERIFY_S(in.size() == tags.size(), "Count doesn't match" + << ": in# " << in.size() + << ", tags# " << tags.size()); + + TVector<TCell> cells(Reserve(in.size())); + for (size_t i = 0; i < in.size(); ++i) { + out.AddTags(tags.at(i)); + cells.emplace_back(in.at(i).AsRef()); + } + + out.SetData(TSerializedCellVec::Serialize(cells)); +} + +void TBaseChangeCollector::SerializeCells(TSerializedCells& out, TArrayRef<const TUpdateOp> in) { + if (!in) { + return; + } + + TVector<TCell> cells(Reserve(in.size())); + for (const auto& op : in) { Y_VERIFY_S(op.Op == ECellOp::Set, "Unexpected cell op: " << op.Op.Raw()); - - out.AddTags(op.Tag); - cells.emplace_back(op.AsCell()); - } - - out.SetData(TSerializedCellVec::Serialize(cells)); -} - -void TBaseChangeCollector::SerializeCells(TSerializedCells& out, const TRowState& state, TArrayRef<const TTag> tags) { - Y_VERIFY_S(state.Size() == tags.size(), "Count doesn't match" - << ": state# " << state.Size() - << ", tags# " << tags.size()); - - TVector<TCell> cells(Reserve(state.Size())); - for (TPos pos = 0; pos < state.Size(); ++pos) { - out.AddTags(tags.at(pos)); - cells.emplace_back(state.Get(pos)); - } - - out.SetData(TSerializedCellVec::Serialize(cells)); -} - + + out.AddTags(op.Tag); + cells.emplace_back(op.AsCell()); + } + + out.SetData(TSerializedCellVec::Serialize(cells)); +} + +void TBaseChangeCollector::SerializeCells(TSerializedCells& out, const TRowState& state, TArrayRef<const TTag> tags) { + Y_VERIFY_S(state.Size() == tags.size(), "Count doesn't match" + << ": state# " << state.Size() + << ", tags# " << tags.size()); + + TVector<TCell> cells(Reserve(state.Size())); + for (TPos pos = 0; pos < state.Size(); ++pos) { + out.AddTags(tags.at(pos)); + cells.emplace_back(state.Get(pos)); + } + + out.SetData(TSerializedCellVec::Serialize(cells)); +} + void TBaseChangeCollector::Serialize(TDataChange& out, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) -{ - SerializeCells(*out.MutableKey(), key, keyTags); - - switch (rop) { + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) +{ + SerializeCells(*out.MutableKey(), key, keyTags); + + switch (rop) { case ERowOp::Upsert: - SerializeCells(*out.MutableUpsert(), updates); - break; + SerializeCells(*out.MutableUpsert(), updates); + break; case ERowOp::Erase: - out.MutableErase(); - break; + out.MutableErase(); + break; case ERowOp::Reset: - SerializeCells(*out.MutableReset(), updates); - break; - default: - Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); - } -} - + SerializeCells(*out.MutableReset(), updates); + break; + default: + Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); + } +} + void TBaseChangeCollector::Serialize(TDataChange& out, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, - const TRowState* oldState, const TRowState* newState, TArrayRef<const TTag> valueTags) -{ - Serialize(out, rop, key, keyTags, {}); - - if (oldState) { - SerializeCells(*out.MutableOldImage(), *oldState, valueTags); - } - - if (newState) { - SerializeCells(*out.MutableNewImage(), *newState, valueTags); - } -} - -void TBaseChangeCollector::Persist(TChangeRecord::EKind kind, const TPathId& pathId, const TDataChange& body) { - NIceDb::TNiceDb db(Db); - - if (!Group) { - Group = Self->AllocateChangeRecordGroup(db); - } - - auto record = TChangeRecordBuilder(kind) - .WithOrder(Self->AllocateChangeRecordOrder(db)) - .WithGroup(*Group) - .WithStep(WriteVersion.Step) - .WithTxId(WriteVersion.TxId) - .WithPathId(pathId) - .WithBody(body.SerializeAsString()) - .Build(); - - Self->PersistChangeRecord(db, record); - Collected.emplace_back(record.GetOrder(), record.GetPathId(), record.GetBody().size()); -} - + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, + const TRowState* oldState, const TRowState* newState, TArrayRef<const TTag> valueTags) +{ + Serialize(out, rop, key, keyTags, {}); + + if (oldState) { + SerializeCells(*out.MutableOldImage(), *oldState, valueTags); + } + + if (newState) { + SerializeCells(*out.MutableNewImage(), *newState, valueTags); + } +} + +void TBaseChangeCollector::Persist(TChangeRecord::EKind kind, const TPathId& pathId, const TDataChange& body) { + NIceDb::TNiceDb db(Db); + + if (!Group) { + Group = Self->AllocateChangeRecordGroup(db); + } + + auto record = TChangeRecordBuilder(kind) + .WithOrder(Self->AllocateChangeRecordOrder(db)) + .WithGroup(*Group) + .WithStep(WriteVersion.Step) + .WithTxId(WriteVersion.TxId) + .WithPathId(pathId) + .WithBody(body.SerializeAsString()) + .Build(); + + Self->PersistChangeRecord(db, record); + Collected.emplace_back(record.GetOrder(), record.GetPathId(), record.GetBody().size()); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_base.h b/ydb/core/tx/datashard/change_collector_base.h index 16df559d7de..bb04deb29c8 100644 --- a/ydb/core/tx/datashard/change_collector_base.h +++ b/ydb/core/tx/datashard/change_collector_base.h @@ -1,58 +1,58 @@ -#pragma once - -#include "change_record.h" - +#pragma once + +#include "change_record.h" + #include <ydb/core/engine/minikql/change_collector_iface.h> #include <ydb/core/protos/change_exchange.pb.h> #include <ydb/core/tablet_flat/flat_database.h> - -#include <util/generic/maybe.h> - -namespace NKikimr { + +#include <util/generic/maybe.h> + +namespace NKikimr { namespace NDataShard { - + class TDataShard; - -class TBaseChangeCollector: public NMiniKQL::IChangeCollector { - using TDataChange = NKikimrChangeExchange::TChangeRecord::TDataChange; - using TSerializedCells = TDataChange::TSerializedCells; - - static void SerializeCells(TSerializedCells& out, TArrayRef<const TRawTypeValue> in, TArrayRef<const NTable::TTag> tags); - static void SerializeCells(TSerializedCells& out, TArrayRef<const NTable::TUpdateOp> in); - static void SerializeCells(TSerializedCells& out, const NTable::TRowState& state, TArrayRef<const NTable::TTag> tags); - -protected: + +class TBaseChangeCollector: public NMiniKQL::IChangeCollector { + using TDataChange = NKikimrChangeExchange::TChangeRecord::TDataChange; + using TSerializedCells = TDataChange::TSerializedCells; + + static void SerializeCells(TSerializedCells& out, TArrayRef<const TRawTypeValue> in, TArrayRef<const NTable::TTag> tags); + static void SerializeCells(TSerializedCells& out, TArrayRef<const NTable::TUpdateOp> in); + static void SerializeCells(TSerializedCells& out, const NTable::TRowState& state, TArrayRef<const NTable::TTag> tags); + +protected: static void Serialize(TDataChange& out, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TUpdateOp> updates); + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TUpdateOp> updates); static void Serialize(TDataChange& out, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, - const NTable::TRowState* oldState, const NTable::TRowState* newState, TArrayRef<const NTable::TTag> valueTags); - - void Persist(TChangeRecord::EKind kind, const TPathId& pathId, const TDataChange& body); - -public: + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, + const NTable::TRowState* oldState, const NTable::TRowState* newState, TArrayRef<const NTable::TTag> valueTags); + + void Persist(TChangeRecord::EKind kind, const TPathId& pathId, const TDataChange& body); + +public: explicit TBaseChangeCollector(TDataShard* self, NTable::TDatabase& db, bool isImmediateTx); - - bool NeedToReadKeys() const override; - void SetReadVersion(const TRowVersion& readVersion) override; - void SetWriteVersion(const TRowVersion& writeVersion) override; - - const TVector<TChange>& GetCollected() const override; - TVector<TChange>&& GetCollected() override; - void Reset() override; - - // there is no Collect, still abstract - -protected: + + bool NeedToReadKeys() const override; + void SetReadVersion(const TRowVersion& readVersion) override; + void SetWriteVersion(const TRowVersion& writeVersion) override; + + const TVector<TChange>& GetCollected() const override; + TVector<TChange>&& GetCollected() override; + void Reset() override; + + // there is no Collect, still abstract + +protected: TDataShard* Self; - NTable::TDatabase& Db; - - TRowVersion WriteVersion; - TMaybe<ui64> Group; - - TVector<TChange> Collected; - -}; // TBaseChangeCollector - + NTable::TDatabase& Db; + + TRowVersion WriteVersion; + TMaybe<ui64> Group; + + TVector<TChange> Collected; + +}; // TBaseChangeCollector + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_cdc_stream.cpp b/ydb/core/tx/datashard/change_collector_cdc_stream.cpp index 6a39e9b43af..f5cb67d1ba4 100644 --- a/ydb/core/tx/datashard/change_collector_cdc_stream.cpp +++ b/ydb/core/tx/datashard/change_collector_cdc_stream.cpp @@ -1,227 +1,227 @@ -#include "change_collector_cdc_stream.h" -#include "datashard_impl.h" - +#include "change_collector_cdc_stream.h" +#include "datashard_impl.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -using namespace NMiniKQL; -using namespace NTable; - -namespace { - - auto MakeValueTags(const TMap<TTag, TUserTable::TUserColumn>& columns) { - TVector<TTag> result; - - for (const auto& [tag, column] : columns) { - if (!column.IsKey) { - result.push_back(tag); - } - } - - return result; - } - - auto MakeTagToPos(TArrayRef<const TTag> tags) { - THashMap<TTag, TPos> result; - - for (TPos pos = 0; pos < tags.size(); ++pos) { - result.emplace(tags.at(pos), pos); - } - - return result; - } - - auto MappedUpdates(TArrayRef<const TUpdateOp> updates) { - THashMap<TTag, TUpdateOp> result; - - for (const auto& op : updates) { - result.emplace(op.Tag, op); - } - - return result; - } - - const TRowState* NullIfErased(const TRowState* state) { - switch (state->GetRowState()) { + +using namespace NMiniKQL; +using namespace NTable; + +namespace { + + auto MakeValueTags(const TMap<TTag, TUserTable::TUserColumn>& columns) { + TVector<TTag> result; + + for (const auto& [tag, column] : columns) { + if (!column.IsKey) { + result.push_back(tag); + } + } + + return result; + } + + auto MakeTagToPos(TArrayRef<const TTag> tags) { + THashMap<TTag, TPos> result; + + for (TPos pos = 0; pos < tags.size(); ++pos) { + result.emplace(tags.at(pos), pos); + } + + return result; + } + + auto MappedUpdates(TArrayRef<const TUpdateOp> updates) { + THashMap<TTag, TUpdateOp> result; + + for (const auto& op : updates) { + result.emplace(op.Tag, op); + } + + return result; + } + + const TRowState* NullIfErased(const TRowState* state) { + switch (state->GetRowState()) { case ERowOp::Upsert: case ERowOp::Reset: - return state; + return state; case ERowOp::Erase: - return nullptr; - default: - Y_FAIL_S("Unexpected row op: " << static_cast<ui8>(state->GetRowState())); - } - } - -} // anonymous - -bool TCdcStreamChangeCollector::NeedToReadKeys() const { - if (CachedNeedToReadKeys) { - return *CachedNeedToReadKeys; - } - - for (const auto& [_, tableInfo] : Self->GetUserTables()) { - for (const auto& [_, streamInfo] : tableInfo->CdcStreams) { - switch (streamInfo.Mode) { + return nullptr; + default: + Y_FAIL_S("Unexpected row op: " << static_cast<ui8>(state->GetRowState())); + } + } + +} // anonymous + +bool TCdcStreamChangeCollector::NeedToReadKeys() const { + if (CachedNeedToReadKeys) { + return *CachedNeedToReadKeys; + } + + for (const auto& [_, tableInfo] : Self->GetUserTables()) { + for (const auto& [_, streamInfo] : tableInfo->CdcStreams) { + switch (streamInfo.Mode) { case NKikimrSchemeOp::ECdcStreamModeKeysOnly: case NKikimrSchemeOp::ECdcStreamModeUpdate: - CachedNeedToReadKeys = false; - break; + CachedNeedToReadKeys = false; + break; case NKikimrSchemeOp::ECdcStreamModeNewImage: case NKikimrSchemeOp::ECdcStreamModeOldImage: case NKikimrSchemeOp::ECdcStreamModeNewAndOldImages: - CachedNeedToReadKeys = true; - break; - default: - Y_FAIL_S("Invalid stream mode: " << static_cast<ui32>(streamInfo.Mode)); - } - } - } - - Y_VERIFY(CachedNeedToReadKeys); - return *CachedNeedToReadKeys; -} - -void TCdcStreamChangeCollector::SetReadVersion(const TRowVersion& readVersion) { - ReadVersion = readVersion; -} - + CachedNeedToReadKeys = true; + break; + default: + Y_FAIL_S("Invalid stream mode: " << static_cast<ui32>(streamInfo.Mode)); + } + } + } + + Y_VERIFY(CachedNeedToReadKeys); + return *CachedNeedToReadKeys; +} + +void TCdcStreamChangeCollector::SetReadVersion(const TRowVersion& readVersion) { + ReadVersion = readVersion; +} + bool TCdcStreamChangeCollector::Collect(const TTableId& tableId, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) -{ - Y_VERIFY_S(Self->IsUserTable(tableId), "Unknown table: " << tableId); - const auto localTableId = Self->GetLocalTableId(tableId); - - auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); - const auto& keyTags = userTable->KeyColumnIds; - - Y_VERIFY_S(key.size() == keyTags.size(), "Count doesn't match" - << ": key# " << key.size() - << ", tags# " << keyTags.size()); - - switch (rop) { + TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) +{ + Y_VERIFY_S(Self->IsUserTable(tableId), "Unknown table: " << tableId); + const auto localTableId = Self->GetLocalTableId(tableId); + + auto userTable = Self->GetUserTables().at(tableId.PathId.LocalPathId); + const auto& keyTags = userTable->KeyColumnIds; + + Y_VERIFY_S(key.size() == keyTags.size(), "Count doesn't match" + << ": key# " << key.size() + << ", tags# " << keyTags.size()); + + switch (rop) { case ERowOp::Upsert: case ERowOp::Erase: case ERowOp::Reset: - break; - default: - Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); - } - - bool read = false; - - for (const auto& [pathId, stream] : userTable->CdcStreams) { - switch (stream.Mode) { + break; + default: + Y_FAIL_S("Unsupported row op: " << static_cast<ui8>(rop)); + } + + bool read = false; + + for (const auto& [pathId, stream] : userTable->CdcStreams) { + switch (stream.Mode) { case NKikimrSchemeOp::ECdcStreamModeKeysOnly: - Persist(pathId, rop, key, keyTags, {}); - break; + Persist(pathId, rop, key, keyTags, {}); + break; case NKikimrSchemeOp::ECdcStreamModeUpdate: - Persist(pathId, rop, key, keyTags, updates); - break; + Persist(pathId, rop, key, keyTags, updates); + break; case NKikimrSchemeOp::ECdcStreamModeNewImage: case NKikimrSchemeOp::ECdcStreamModeOldImage: case NKikimrSchemeOp::ECdcStreamModeNewAndOldImages: { - const auto valueTags = MakeValueTags(userTable->Columns); - const auto oldState = GetCurrentState(localTableId, key, keyTags, valueTags); - - if (!oldState) { - return false; - } - + const auto valueTags = MakeValueTags(userTable->Columns); + const auto oldState = GetCurrentState(localTableId, key, keyTags, valueTags); + + if (!oldState) { + return false; + } + if (stream.Mode == NKikimrSchemeOp::ECdcStreamModeOldImage) { - Persist(pathId, rop, key, keyTags, NullIfErased(&*oldState), nullptr, valueTags); - } else { - const auto newState = PatchState(*oldState, rop, MakeTagToPos(valueTags), MappedUpdates(updates)); - + Persist(pathId, rop, key, keyTags, NullIfErased(&*oldState), nullptr, valueTags); + } else { + const auto newState = PatchState(*oldState, rop, MakeTagToPos(valueTags), MappedUpdates(updates)); + if (stream.Mode == NKikimrSchemeOp::ECdcStreamModeNewImage) { - Persist(pathId, rop, key, keyTags, nullptr, NullIfErased(&newState), valueTags); - } else { - Persist(pathId, rop, key, keyTags, NullIfErased(&*oldState), NullIfErased(&newState), valueTags); - } - } - - read = true; - break; - } - default: - Y_FAIL_S("Invalid stream mode: " << static_cast<ui32>(stream.Mode)); - } - } - - if (read) { - RowsCache.UpdateCachedRow(localTableId, rop, key, updates); - } - - return true; -} - -TMaybe<TRowState> TCdcStreamChangeCollector::GetCurrentState(ui32 tid, TArrayRef<const TRawTypeValue> key, - TArrayRef<const TTag> keyTags, TArrayRef<const TTag> valueTags) -{ - TRowState row; - const auto ready = RowsCache.SelectRow(Db, tid, key, MakeTagToPos(keyTags), valueTags, row, ReadVersion); - - if (ready == EReady::Page) { - return Nothing(); - } - - return row; -} - + Persist(pathId, rop, key, keyTags, nullptr, NullIfErased(&newState), valueTags); + } else { + Persist(pathId, rop, key, keyTags, NullIfErased(&*oldState), NullIfErased(&newState), valueTags); + } + } + + read = true; + break; + } + default: + Y_FAIL_S("Invalid stream mode: " << static_cast<ui32>(stream.Mode)); + } + } + + if (read) { + RowsCache.UpdateCachedRow(localTableId, rop, key, updates); + } + + return true; +} + +TMaybe<TRowState> TCdcStreamChangeCollector::GetCurrentState(ui32 tid, TArrayRef<const TRawTypeValue> key, + TArrayRef<const TTag> keyTags, TArrayRef<const TTag> valueTags) +{ + TRowState row; + const auto ready = RowsCache.SelectRow(Db, tid, key, MakeTagToPos(keyTags), valueTags, row, ReadVersion); + + if (ready == EReady::Page) { + return Nothing(); + } + + return row; +} + TRowState TCdcStreamChangeCollector::PatchState(const TRowState& oldState, ERowOp rop, - const THashMap<TTag, TPos>& tagToPos, const THashMap<TTag, TUpdateOp>& updates) -{ - TRowState newState; - - newState.Init(tagToPos.size()); - newState.Touch(rop); - - switch (rop) { + const THashMap<TTag, TPos>& tagToPos, const THashMap<TTag, TUpdateOp>& updates) +{ + TRowState newState; + + newState.Init(tagToPos.size()); + newState.Touch(rop); + + switch (rop) { case ERowOp::Upsert: case ERowOp::Reset: - for (const auto [tag, pos] : tagToPos) { - auto it = updates.find(tag); - if (it != updates.end()) { - newState.Set(pos, it->second.Op, it->second.AsCell()); + for (const auto [tag, pos] : tagToPos) { + auto it = updates.find(tag); + if (it != updates.end()) { + newState.Set(pos, it->second.Op, it->second.AsCell()); } else if (rop == ERowOp::Upsert) { - newState.Set(pos, oldState.GetOp(pos), oldState.Get(pos)); - } else { + newState.Set(pos, oldState.GetOp(pos), oldState.Get(pos)); + } else { newState.Set(pos, ECellOp::Null, TCell()); - } - } - break; + } + } + break; case ERowOp::Erase: - break; - default: - Y_FAIL("unreachable"); - } - - Y_VERIFY(newState.IsFinalized()); - return newState; -} - + break; + default: + Y_FAIL("unreachable"); + } + + Y_VERIFY(newState.IsFinalized()); + return newState; +} + void TCdcStreamChangeCollector::Persist(const TPathId& pathId, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) -{ - NKikimrChangeExchange::TChangeRecord::TDataChange body; - Serialize(body, rop, key, keyTags, updates); - TBaseChangeCollector::Persist(TChangeRecord::EKind::CdcDataChange, pathId, body); -} - + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, TArrayRef<const TUpdateOp> updates) +{ + NKikimrChangeExchange::TChangeRecord::TDataChange body; + Serialize(body, rop, key, keyTags, updates); + TBaseChangeCollector::Persist(TChangeRecord::EKind::CdcDataChange, pathId, body); +} + void TCdcStreamChangeCollector::Persist(const TPathId& pathId, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, - const TRowState* oldState, const TRowState* newState, TArrayRef<const TTag> valueTags) -{ - if (!oldState && !newState) { - return; - } - - NKikimrChangeExchange::TChangeRecord::TDataChange body; - Serialize(body, rop, key, keyTags, oldState, newState, valueTags); - TBaseChangeCollector::Persist(TChangeRecord::EKind::CdcDataChange, pathId, body); -} - + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> keyTags, + const TRowState* oldState, const TRowState* newState, TArrayRef<const TTag> valueTags) +{ + if (!oldState && !newState) { + return; + } + + NKikimrChangeExchange::TChangeRecord::TDataChange body; + Serialize(body, rop, key, keyTags, oldState, newState, valueTags); + TBaseChangeCollector::Persist(TChangeRecord::EKind::CdcDataChange, pathId, body); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_cdc_stream.h b/ydb/core/tx/datashard/change_collector_cdc_stream.h index 5cf6eb4292c..e02382d5dbd 100644 --- a/ydb/core/tx/datashard/change_collector_cdc_stream.h +++ b/ydb/core/tx/datashard/change_collector_cdc_stream.h @@ -1,40 +1,40 @@ -#pragma once - -#include "change_collector_base.h" -#include "change_collector_helpers.h" -#include "datashard_user_table.h" - -namespace NKikimr { +#pragma once + +#include "change_collector_base.h" +#include "change_collector_helpers.h" +#include "datashard_user_table.h" + +namespace NKikimr { namespace NDataShard { - -class TCdcStreamChangeCollector: public TBaseChangeCollector { - TMaybe<NTable::TRowState> GetCurrentState(ui32 tid, TArrayRef<const TRawTypeValue> key, - TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TTag> valueTags); + +class TCdcStreamChangeCollector: public TBaseChangeCollector { + TMaybe<NTable::TRowState> GetCurrentState(ui32 tid, TArrayRef<const TRawTypeValue> key, + TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TTag> valueTags); static NTable::TRowState PatchState(const NTable::TRowState& oldState, NTable::ERowOp rop, - const THashMap<NTable::TTag, NTable::TPos>& tagToPos, const THashMap<NTable::TTag, NTable::TUpdateOp>& updates); - + const THashMap<NTable::TTag, NTable::TPos>& tagToPos, const THashMap<NTable::TTag, NTable::TUpdateOp>& updates); + void Persist(const TPathId& pathId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TUpdateOp> updates); + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, TArrayRef<const NTable::TUpdateOp> updates); void Persist(const TPathId& pathId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, - const NTable::TRowState* oldState, const NTable::TRowState* newState, TArrayRef<const NTable::TTag> valueTags); - -public: - using TBaseChangeCollector::TBaseChangeCollector; - - bool NeedToReadKeys() const override; - void SetReadVersion(const TRowVersion& readVersion) override; - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, + const NTable::TRowState* oldState, const NTable::TRowState* newState, TArrayRef<const NTable::TTag> valueTags); + +public: + using TBaseChangeCollector::TBaseChangeCollector; + + bool NeedToReadKeys() const override; + void SetReadVersion(const TRowVersion& readVersion) override; + bool Collect(const TTableId& tableId, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override; - -private: - TRowVersion ReadVersion; - - mutable TMaybe<bool> CachedNeedToReadKeys; - TRowsCache RowsCache; - -}; // TCdcStreamChangeCollector - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates) override; + +private: + TRowVersion ReadVersion; + + mutable TMaybe<bool> CachedNeedToReadKeys; + TRowsCache RowsCache; + +}; // TCdcStreamChangeCollector + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_helpers.cpp b/ydb/core/tx/datashard/change_collector_helpers.cpp index 1a8504a536a..fce849a0764 100644 --- a/ydb/core/tx/datashard/change_collector_helpers.cpp +++ b/ydb/core/tx/datashard/change_collector_helpers.cpp @@ -1,144 +1,144 @@ -#include "change_collector_helpers.h" - -namespace NKikimr { +#include "change_collector_helpers.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTable; - -namespace { - - TString SerializeKey(TArrayRef<const TRawTypeValue> key) { - TVector<TCell> cells(Reserve(key.size())); - for (const auto& k : key) { - cells.emplace_back(k.AsRef()); - } - - return TSerializedCellVec::Serialize(cells); - } - -} // anonymous - -EReady TRowsCache::SelectRow(TDatabase& db, ui32 tid, - TArrayRef<const TRawTypeValue> key, const THashMap<TTag, TPos>& keyTagToPos, - TArrayRef<const TTag> tags, TRowState& rowState, const TRowVersion& readVersion) -{ - const auto* row = FindCachedRow(tid, key); - - if (!row) { - TRowState selected; - const auto ready = db.Select(tid, key, tags, selected, 0, readVersion); - - if (ready == EReady::Page) { - return ready; - } - - row = CacheRow(tid, ready, key, tags, selected); - } - - rowState.Init(tags.size()); - rowState.Touch(row->Rop); - - for (TPos pos = 0; pos < tags.size(); ++pos) { - auto it = keyTagToPos.find(tags.at(pos)); - if (it == keyTagToPos.end()) { - continue; - } - - Y_VERIFY(it->second < key.size()); + +using namespace NTable; + +namespace { + + TString SerializeKey(TArrayRef<const TRawTypeValue> key) { + TVector<TCell> cells(Reserve(key.size())); + for (const auto& k : key) { + cells.emplace_back(k.AsRef()); + } + + return TSerializedCellVec::Serialize(cells); + } + +} // anonymous + +EReady TRowsCache::SelectRow(TDatabase& db, ui32 tid, + TArrayRef<const TRawTypeValue> key, const THashMap<TTag, TPos>& keyTagToPos, + TArrayRef<const TTag> tags, TRowState& rowState, const TRowVersion& readVersion) +{ + const auto* row = FindCachedRow(tid, key); + + if (!row) { + TRowState selected; + const auto ready = db.Select(tid, key, tags, selected, 0, readVersion); + + if (ready == EReady::Page) { + return ready; + } + + row = CacheRow(tid, ready, key, tags, selected); + } + + rowState.Init(tags.size()); + rowState.Touch(row->Rop); + + for (TPos pos = 0; pos < tags.size(); ++pos) { + auto it = keyTagToPos.find(tags.at(pos)); + if (it == keyTagToPos.end()) { + continue; + } + + Y_VERIFY(it->second < key.size()); rowState.Set(pos, ECellOp::Set, key.at(it->second).AsRef()); - } - - switch (row->Ready) { - case EReady::Data: - switch (row->Rop) { + } + + switch (row->Ready) { + case EReady::Data: + switch (row->Rop) { case ERowOp::Upsert: case ERowOp::Reset: - for (TPos pos = 0; pos < tags.size(); ++pos) { - const auto tag = tags.at(pos); - if (keyTagToPos.contains(tag)) { - continue; - } - - if (const auto* cell = row->Cells.FindPtr(tag)) { + for (TPos pos = 0; pos < tags.size(); ++pos) { + const auto tag = tags.at(pos); + if (keyTagToPos.contains(tag)) { + continue; + } + + if (const auto* cell = row->Cells.FindPtr(tag)) { rowState.Set(pos, ECellOp::Set, TCell(cell->data(), cell->size())); - } else { + } else { rowState.Set(pos, ECellOp::Null, TCell()); - } - } - break; - default: - Y_FAIL("unreachable"); - } - break; - case EReady::Gone: - break; - default: - Y_FAIL("unreachable"); - } - - Y_VERIFY(rowState.IsFinalized()); - return row->Ready; -} - + } + } + break; + default: + Y_FAIL("unreachable"); + } + break; + case EReady::Gone: + break; + default: + Y_FAIL("unreachable"); + } + + Y_VERIFY(rowState.IsFinalized()); + return row->Ready; +} + void TRowsCache::UpdateCachedRow(ui32 tid, ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) -{ - auto& row = Rows[tid][SerializeKey(key)]; - - row.Rop = rop; + TArrayRef<const TRawTypeValue> key, TArrayRef<const TUpdateOp> updates) +{ + auto& row = Rows[tid][SerializeKey(key)]; + + row.Rop = rop; if (rop == ERowOp::Erase) { - row.Ready = EReady::Gone; - row.Cells.clear(); - return; - } - - row.Ready = EReady::Data; + row.Ready = EReady::Gone; + row.Cells.clear(); + return; + } + + row.Ready = EReady::Data; if (rop == ERowOp::Reset) { - row.Cells.clear(); - } - - for (const auto& update : updates) { - if (update.Value.IsEmpty()) { - continue; - } - - row.Cells[update.Tag] = update.Value.ToStringBuf(); - } -} - -const TRowsCache::TRow* TRowsCache::CacheRow(ui32 tid, EReady ready, - TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> tags, const TRowState& rowState) -{ - Y_VERIFY(ready != EReady::Page); - Y_VERIFY(tags.size() == rowState.Size()); - - auto& row = Rows[tid][SerializeKey(key)]; - - row.Ready = ready; - if (ready == EReady::Gone) { + row.Cells.clear(); + } + + for (const auto& update : updates) { + if (update.Value.IsEmpty()) { + continue; + } + + row.Cells[update.Tag] = update.Value.ToStringBuf(); + } +} + +const TRowsCache::TRow* TRowsCache::CacheRow(ui32 tid, EReady ready, + TArrayRef<const TRawTypeValue> key, TArrayRef<const TTag> tags, const TRowState& rowState) +{ + Y_VERIFY(ready != EReady::Page); + Y_VERIFY(tags.size() == rowState.Size()); + + auto& row = Rows[tid][SerializeKey(key)]; + + row.Ready = ready; + if (ready == EReady::Gone) { row.Rop = ERowOp::Erase; - return &row; - } - - row.Rop = rowState.GetRowState(); - for (TPos pos = 0; pos < tags.size(); ++pos) { - const auto& cell = rowState.Get(pos); - if (cell.IsNull()) { - continue; - } - - row.Cells[tags.at(pos)] = cell.AsBuf(); - } - - return &row; -} - -const TRowsCache::TRow* TRowsCache::FindCachedRow(ui32 tid, TArrayRef<const TRawTypeValue> key) const { - const auto* cached = Rows.FindPtr(tid); - return cached - ? cached->FindPtr(SerializeKey(key)) - : nullptr; -} - + return &row; + } + + row.Rop = rowState.GetRowState(); + for (TPos pos = 0; pos < tags.size(); ++pos) { + const auto& cell = rowState.Get(pos); + if (cell.IsNull()) { + continue; + } + + row.Cells[tags.at(pos)] = cell.AsBuf(); + } + + return &row; +} + +const TRowsCache::TRow* TRowsCache::FindCachedRow(ui32 tid, TArrayRef<const TRawTypeValue> key) const { + const auto* cached = Rows.FindPtr(tid); + return cached + ? cached->FindPtr(SerializeKey(key)) + : nullptr; +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_collector_helpers.h b/ydb/core/tx/datashard/change_collector_helpers.h index 7b81ab5442b..b41c5986f27 100644 --- a/ydb/core/tx/datashard/change_collector_helpers.h +++ b/ydb/core/tx/datashard/change_collector_helpers.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include <ydb/core/tablet_flat/flat_database.h> - -#include <util/generic/hash.h> - -namespace NKikimr { + +#include <util/generic/hash.h> + +namespace NKikimr { namespace NDataShard { - -class TRowsCache { - struct TRow { - NTable::EReady Ready; + +class TRowsCache { + struct TRow { + NTable::EReady Ready; NTable::ERowOp Rop; - THashMap<NTable::TTag, TString> Cells; - }; - - const TRow* CacheRow(ui32 tid, NTable::EReady ready, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> tags, const NTable::TRowState& rowState); - - const TRow* FindCachedRow(ui32 tid, TArrayRef<const TRawTypeValue> key) const; - -public: - NTable::EReady SelectRow(NTable::TDatabase& db, ui32 tid, - TArrayRef<const TRawTypeValue> key, const THashMap<NTable::TTag, NTable::TPos>& keyTagToPos, - TArrayRef<const NTable::TTag> tags, NTable::TRowState& rowState, const TRowVersion& readVersion); - + THashMap<NTable::TTag, TString> Cells; + }; + + const TRow* CacheRow(ui32 tid, NTable::EReady ready, + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> tags, const NTable::TRowState& rowState); + + const TRow* FindCachedRow(ui32 tid, TArrayRef<const TRawTypeValue> key) const; + +public: + NTable::EReady SelectRow(NTable::TDatabase& db, ui32 tid, + TArrayRef<const TRawTypeValue> key, const THashMap<NTable::TTag, NTable::TPos>& keyTagToPos, + TArrayRef<const NTable::TTag> tags, NTable::TRowState& rowState, const TRowVersion& readVersion); + void UpdateCachedRow(ui32 tid, NTable::ERowOp rop, - TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates); - -private: - THashMap<ui32, THashMap<TString, TRow>> Rows; - -}; // TRowsCache - + TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TUpdateOp> updates); + +private: + THashMap<ui32, THashMap<TString, TRow>> Rows; + +}; // TRowsCache + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_exchange.cpp b/ydb/core/tx/datashard/change_exchange.cpp index 7adba1f9178..99270fd69a8 100644 --- a/ydb/core/tx/datashard/change_exchange.cpp +++ b/ydb/core/tx/datashard/change_exchange.cpp @@ -1,155 +1,155 @@ -#include "change_exchange.h" - -#include <util/string/builder.h> -#include <util/string/join.h> - -namespace NKikimr { +#include "change_exchange.h" + +#include <util/string/builder.h> +#include <util/string/join.h> + +namespace NKikimr { namespace NDataShard { - -/// TEvEnqueueRecords -TEvChangeExchange::TEvEnqueueRecords::TEvEnqueueRecords(const TVector<TRecordInfo>& records) - : Records(records) -{ -} - -TEvChangeExchange::TEvEnqueueRecords::TEvEnqueueRecords(TVector<TRecordInfo>&& records) - : Records(std::move(records)) -{ -} - -TString TEvChangeExchange::TEvEnqueueRecords::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Records [" << JoinSeq(",", Records) << "]" - << " }"; -} - -TEvChangeExchange::TEvEnqueueRecords::TRecordInfo::TRecordInfo(ui64 order, const TPathId& pathId, ui64 bodySize) - : Order(order) - , PathId(pathId) - , BodySize(bodySize) -{ -} - -void TEvChangeExchange::TEvEnqueueRecords::TRecordInfo::Out(IOutputStream& out) const { - out << "{" - << " Order: " << Order - << " PathId: " << PathId - << " BodySize: " << BodySize - << " }"; -} - -/// TEvRequestRecords -TEvChangeExchange::TEvRequestRecords::TEvRequestRecords(const TVector<TRecordInfo>& records) - : Records(records) -{ -} - -TEvChangeExchange::TEvRequestRecords::TEvRequestRecords(TVector<TRecordInfo>&& records) - : Records(std::move(records)) -{ -} - -TString TEvChangeExchange::TEvRequestRecords::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Records [" << JoinSeq(",", Records) << "]" - << " }"; -} - -TEvChangeExchange::TEvRequestRecords::TRecordInfo::TRecordInfo(ui64 order, ui64 bodySize) - : Order(order) - , BodySize(bodySize) -{ -} - -bool TEvChangeExchange::TEvRequestRecords::TRecordInfo::operator<(const TRecordInfo& rhs) const { - return Order < rhs.Order; -} - -void TEvChangeExchange::TEvRequestRecords::TRecordInfo::Out(IOutputStream& out) const { - out << "{" - << " Order: " << Order - << " BodySize: " << BodySize - << " }"; -} - -/// TEvRemoveRecords -TEvChangeExchange::TEvRemoveRecords::TEvRemoveRecords(const TVector<ui64>& records) - : Records(records) -{ -} - -TEvChangeExchange::TEvRemoveRecords::TEvRemoveRecords(TVector<ui64>&& records) - : Records(std::move(records)) -{ -} - -TString TEvChangeExchange::TEvRemoveRecords::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Records [" << JoinSeq(",", Records) << "]" - << " }"; -} - -/// TEvRecords -TEvChangeExchange::TEvRecords::TEvRecords(const TVector<TChangeRecord>& records) - : Records(records) -{ -} - -TEvChangeExchange::TEvRecords::TEvRecords(TVector<TChangeRecord>&& records) - : Records(std::move(records)) -{ -} - -TString TEvChangeExchange::TEvRecords::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Records [" << JoinSeq(",", Records) << "]" - << " }"; -} - -/// TEvForgetRecords -TEvChangeExchange::TEvForgetRecords::TEvForgetRecords(const TVector<ui64>& records) - : Records(records) -{ -} - -TEvChangeExchange::TEvForgetRecords::TEvForgetRecords(TVector<ui64>&& records) - : Records(std::move(records)) -{ -} - -TString TEvChangeExchange::TEvForgetRecords::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " Records [" << JoinSeq(",", Records) << "]" - << " }"; -} - -/// TEvAddSender -TEvChangeExchange::TEvAddSender::TEvAddSender(const TTableId& userTableId, TEvChangeExchange::ESenderType type, const TPathId& pathId) - : UserTableId(userTableId) - , Type(type) - , PathId(pathId) -{ -} - -TString TEvChangeExchange::TEvAddSender::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " UserTableId: " << UserTableId - << " Type: " << Type - << " PathId: " << PathId - << " }"; -} - -/// TEvRemoveSender -TEvChangeExchange::TEvRemoveSender::TEvRemoveSender(const TPathId& pathId) - : PathId(pathId) -{ -} - -TString TEvChangeExchange::TEvRemoveSender::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " PathId: " << PathId - << " }"; -} - + +/// TEvEnqueueRecords +TEvChangeExchange::TEvEnqueueRecords::TEvEnqueueRecords(const TVector<TRecordInfo>& records) + : Records(records) +{ +} + +TEvChangeExchange::TEvEnqueueRecords::TEvEnqueueRecords(TVector<TRecordInfo>&& records) + : Records(std::move(records)) +{ +} + +TString TEvChangeExchange::TEvEnqueueRecords::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Records [" << JoinSeq(",", Records) << "]" + << " }"; +} + +TEvChangeExchange::TEvEnqueueRecords::TRecordInfo::TRecordInfo(ui64 order, const TPathId& pathId, ui64 bodySize) + : Order(order) + , PathId(pathId) + , BodySize(bodySize) +{ +} + +void TEvChangeExchange::TEvEnqueueRecords::TRecordInfo::Out(IOutputStream& out) const { + out << "{" + << " Order: " << Order + << " PathId: " << PathId + << " BodySize: " << BodySize + << " }"; +} + +/// TEvRequestRecords +TEvChangeExchange::TEvRequestRecords::TEvRequestRecords(const TVector<TRecordInfo>& records) + : Records(records) +{ +} + +TEvChangeExchange::TEvRequestRecords::TEvRequestRecords(TVector<TRecordInfo>&& records) + : Records(std::move(records)) +{ +} + +TString TEvChangeExchange::TEvRequestRecords::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Records [" << JoinSeq(",", Records) << "]" + << " }"; +} + +TEvChangeExchange::TEvRequestRecords::TRecordInfo::TRecordInfo(ui64 order, ui64 bodySize) + : Order(order) + , BodySize(bodySize) +{ +} + +bool TEvChangeExchange::TEvRequestRecords::TRecordInfo::operator<(const TRecordInfo& rhs) const { + return Order < rhs.Order; +} + +void TEvChangeExchange::TEvRequestRecords::TRecordInfo::Out(IOutputStream& out) const { + out << "{" + << " Order: " << Order + << " BodySize: " << BodySize + << " }"; +} + +/// TEvRemoveRecords +TEvChangeExchange::TEvRemoveRecords::TEvRemoveRecords(const TVector<ui64>& records) + : Records(records) +{ +} + +TEvChangeExchange::TEvRemoveRecords::TEvRemoveRecords(TVector<ui64>&& records) + : Records(std::move(records)) +{ +} + +TString TEvChangeExchange::TEvRemoveRecords::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Records [" << JoinSeq(",", Records) << "]" + << " }"; +} + +/// TEvRecords +TEvChangeExchange::TEvRecords::TEvRecords(const TVector<TChangeRecord>& records) + : Records(records) +{ +} + +TEvChangeExchange::TEvRecords::TEvRecords(TVector<TChangeRecord>&& records) + : Records(std::move(records)) +{ +} + +TString TEvChangeExchange::TEvRecords::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Records [" << JoinSeq(",", Records) << "]" + << " }"; +} + +/// TEvForgetRecords +TEvChangeExchange::TEvForgetRecords::TEvForgetRecords(const TVector<ui64>& records) + : Records(records) +{ +} + +TEvChangeExchange::TEvForgetRecords::TEvForgetRecords(TVector<ui64>&& records) + : Records(std::move(records)) +{ +} + +TString TEvChangeExchange::TEvForgetRecords::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " Records [" << JoinSeq(",", Records) << "]" + << " }"; +} + +/// TEvAddSender +TEvChangeExchange::TEvAddSender::TEvAddSender(const TTableId& userTableId, TEvChangeExchange::ESenderType type, const TPathId& pathId) + : UserTableId(userTableId) + , Type(type) + , PathId(pathId) +{ +} + +TString TEvChangeExchange::TEvAddSender::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " UserTableId: " << UserTableId + << " Type: " << Type + << " PathId: " << PathId + << " }"; +} + +/// TEvRemoveSender +TEvChangeExchange::TEvRemoveSender::TEvRemoveSender(const TPathId& pathId) + : PathId(pathId) +{ +} + +TString TEvChangeExchange::TEvRemoveSender::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " PathId: " << PathId + << " }"; +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_exchange.h b/ydb/core/tx/datashard/change_exchange.h index 62c34e8c011..93fb3617b15 100644 --- a/ydb/core/tx/datashard/change_exchange.h +++ b/ydb/core/tx/datashard/change_exchange.h @@ -1,161 +1,161 @@ -#pragma once - -#include "defs.h" -#include "change_record.h" - +#pragma once + +#include "defs.h" +#include "change_record.h" + #include <ydb/core/protos/change_exchange.pb.h> #include <ydb/core/scheme/scheme_tabledefs.h> - -#include <util/generic/vector.h> - -namespace NKikimr { + +#include <util/generic/vector.h> + +namespace NKikimr { namespace NDataShard { - + class TDataShard; - -struct TEvChangeExchange { - enum EEv { - /// Network exchange protocol - // Handshake between sender & receiver - EvHandshake = EventSpaceBegin(TKikimrEvents::ES_CHANGE_EXCHANGE), - // Apply change record(s) on receiver - EvApplyRecords, - // Handshake & application status - EvStatus, - // Activation - EvActivateSender, - EvActivateSenderAck, - - /// Local exchange (mostly using change's id) - // Enqueue for sending - EvEnqueueRecords, - // Request change record(s) by id - EvRequestRecords, - // Change record(s) - EvRecords, - // Remove change record(s) from local database - EvRemoveRecords, - - // Add new change sender - EvAddSender, - // Remove existing change sender - EvRemoveSender, - - // Already removed records that the sender should forget about - EvForgetRecods, - - // Split/merge - EvSplitAck, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_CHANGE_EXCHANGE)); - - /// Network events - struct TEvHandshake: public TEventPB<TEvHandshake, NKikimrChangeExchange::TEvHandshake, EvHandshake> {}; - struct TEvApplyRecords: public TEventPB<TEvApplyRecords, NKikimrChangeExchange::TEvApplyRecords, EvApplyRecords> {}; - struct TEvStatus: public TEventPB<TEvStatus, NKikimrChangeExchange::TEvStatus, EvStatus> {}; - struct TEvActivateSender: public TEventPB<TEvActivateSender, NKikimrChangeExchange::TEvActivateSender, EvActivateSender> {}; - struct TEvActivateSenderAck: public TEventPB<TEvActivateSenderAck, NKikimrChangeExchange::TEvActivateSenderAck, EvActivateSenderAck> {}; - - /// Local events - struct TEvEnqueueRecords: public TEventLocal<TEvEnqueueRecords, EvEnqueueRecords> { - struct TRecordInfo { - ui64 Order; - TPathId PathId; - ui64 BodySize; - - TRecordInfo(ui64 order, const TPathId& pathId, ui64 bodySize); - - void Out(IOutputStream& out) const; - }; - - TVector<TRecordInfo> Records; - - explicit TEvEnqueueRecords(const TVector<TRecordInfo>& records); - explicit TEvEnqueueRecords(TVector<TRecordInfo>&& records); - TString ToString() const override; - }; - - struct TEvRequestRecords: public TEventLocal<TEvRequestRecords, EvRequestRecords> { - struct TRecordInfo { - ui64 Order; - ui64 BodySize; - - TRecordInfo(ui64 order, ui64 bodySize = 0); - - bool operator<(const TRecordInfo& rhs) const; - void Out(IOutputStream& out) const; - }; - - TVector<TRecordInfo> Records; - - explicit TEvRequestRecords(const TVector<TRecordInfo>& records); - explicit TEvRequestRecords(TVector<TRecordInfo>&& records); - TString ToString() const override; - }; - - struct TEvRemoveRecords: public TEventLocal<TEvRemoveRecords, EvRemoveRecords> { - TVector<ui64> Records; - - explicit TEvRemoveRecords(const TVector<ui64>& records); - explicit TEvRemoveRecords(TVector<ui64>&& records); - TString ToString() const override; - }; - - struct TEvRecords: public TEventLocal<TEvRecords, EvRecords> { - TVector<TChangeRecord> Records; - - explicit TEvRecords(const TVector<TChangeRecord>& records); - explicit TEvRecords(TVector<TChangeRecord>&& records); - TString ToString() const override; - }; - - struct TEvForgetRecords: public TEventLocal<TEvForgetRecords, EvForgetRecods> { - TVector<ui64> Records; - - explicit TEvForgetRecords(const TVector<ui64>& records); - explicit TEvForgetRecords(TVector<ui64>&& records); - TString ToString() const override; - }; - - enum class ESenderType { - AsyncIndex, - CdcStream, - }; - - struct TEvAddSender: public TEventLocal<TEvAddSender, EvAddSender> { - TTableId UserTableId; - ESenderType Type; - TPathId PathId; - - explicit TEvAddSender(const TTableId& userTableId, ESenderType type, const TPathId& pathId); - TString ToString() const override; - }; - - struct TEvRemoveSender: public TEventLocal<TEvRemoveSender, EvRemoveSender> { - TPathId PathId; - - explicit TEvRemoveSender(const TPathId& pathId); - TString ToString() const override; - }; - - struct TEvSplitAck: public TEventLocal<TEvSplitAck, EvSplitAck> { - }; - -}; // TEvChangeExchange - + +struct TEvChangeExchange { + enum EEv { + /// Network exchange protocol + // Handshake between sender & receiver + EvHandshake = EventSpaceBegin(TKikimrEvents::ES_CHANGE_EXCHANGE), + // Apply change record(s) on receiver + EvApplyRecords, + // Handshake & application status + EvStatus, + // Activation + EvActivateSender, + EvActivateSenderAck, + + /// Local exchange (mostly using change's id) + // Enqueue for sending + EvEnqueueRecords, + // Request change record(s) by id + EvRequestRecords, + // Change record(s) + EvRecords, + // Remove change record(s) from local database + EvRemoveRecords, + + // Add new change sender + EvAddSender, + // Remove existing change sender + EvRemoveSender, + + // Already removed records that the sender should forget about + EvForgetRecods, + + // Split/merge + EvSplitAck, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_CHANGE_EXCHANGE)); + + /// Network events + struct TEvHandshake: public TEventPB<TEvHandshake, NKikimrChangeExchange::TEvHandshake, EvHandshake> {}; + struct TEvApplyRecords: public TEventPB<TEvApplyRecords, NKikimrChangeExchange::TEvApplyRecords, EvApplyRecords> {}; + struct TEvStatus: public TEventPB<TEvStatus, NKikimrChangeExchange::TEvStatus, EvStatus> {}; + struct TEvActivateSender: public TEventPB<TEvActivateSender, NKikimrChangeExchange::TEvActivateSender, EvActivateSender> {}; + struct TEvActivateSenderAck: public TEventPB<TEvActivateSenderAck, NKikimrChangeExchange::TEvActivateSenderAck, EvActivateSenderAck> {}; + + /// Local events + struct TEvEnqueueRecords: public TEventLocal<TEvEnqueueRecords, EvEnqueueRecords> { + struct TRecordInfo { + ui64 Order; + TPathId PathId; + ui64 BodySize; + + TRecordInfo(ui64 order, const TPathId& pathId, ui64 bodySize); + + void Out(IOutputStream& out) const; + }; + + TVector<TRecordInfo> Records; + + explicit TEvEnqueueRecords(const TVector<TRecordInfo>& records); + explicit TEvEnqueueRecords(TVector<TRecordInfo>&& records); + TString ToString() const override; + }; + + struct TEvRequestRecords: public TEventLocal<TEvRequestRecords, EvRequestRecords> { + struct TRecordInfo { + ui64 Order; + ui64 BodySize; + + TRecordInfo(ui64 order, ui64 bodySize = 0); + + bool operator<(const TRecordInfo& rhs) const; + void Out(IOutputStream& out) const; + }; + + TVector<TRecordInfo> Records; + + explicit TEvRequestRecords(const TVector<TRecordInfo>& records); + explicit TEvRequestRecords(TVector<TRecordInfo>&& records); + TString ToString() const override; + }; + + struct TEvRemoveRecords: public TEventLocal<TEvRemoveRecords, EvRemoveRecords> { + TVector<ui64> Records; + + explicit TEvRemoveRecords(const TVector<ui64>& records); + explicit TEvRemoveRecords(TVector<ui64>&& records); + TString ToString() const override; + }; + + struct TEvRecords: public TEventLocal<TEvRecords, EvRecords> { + TVector<TChangeRecord> Records; + + explicit TEvRecords(const TVector<TChangeRecord>& records); + explicit TEvRecords(TVector<TChangeRecord>&& records); + TString ToString() const override; + }; + + struct TEvForgetRecords: public TEventLocal<TEvForgetRecords, EvForgetRecods> { + TVector<ui64> Records; + + explicit TEvForgetRecords(const TVector<ui64>& records); + explicit TEvForgetRecords(TVector<ui64>&& records); + TString ToString() const override; + }; + + enum class ESenderType { + AsyncIndex, + CdcStream, + }; + + struct TEvAddSender: public TEventLocal<TEvAddSender, EvAddSender> { + TTableId UserTableId; + ESenderType Type; + TPathId PathId; + + explicit TEvAddSender(const TTableId& userTableId, ESenderType type, const TPathId& pathId); + TString ToString() const override; + }; + + struct TEvRemoveSender: public TEventLocal<TEvRemoveSender, EvRemoveSender> { + TPathId PathId; + + explicit TEvRemoveSender(const TPathId& pathId); + TString ToString() const override; + }; + + struct TEvSplitAck: public TEventLocal<TEvSplitAck, EvSplitAck> { + }; + +}; // TEvChangeExchange + IActor* CreateChangeSender(const TDataShard* self); -IActor* CreateChangeExchangeSplit(const TDataShard* self, const TVector<ui64>& dstDataShards); - +IActor* CreateChangeExchangeSplit(const TDataShard* self, const TVector<ui64>& dstDataShards); + } // NDataShard -} // NKikimr - +} // NKikimr + Y_DECLARE_OUT_SPEC(inline, NKikimr::NDataShard::TEvChangeExchange::TEvEnqueueRecords::TRecordInfo, o, x) { - return x.Out(o); -} - + return x.Out(o); +} + Y_DECLARE_OUT_SPEC(inline, NKikimr::NDataShard::TEvChangeExchange::TEvRequestRecords::TRecordInfo, o, x) { - return x.Out(o); -} + return x.Out(o); +} diff --git a/ydb/core/tx/datashard/change_exchange_helpers.h b/ydb/core/tx/datashard/change_exchange_helpers.h index f9aea96759e..7383fe6a620 100644 --- a/ydb/core/tx/datashard/change_exchange_helpers.h +++ b/ydb/core/tx/datashard/change_exchange_helpers.h @@ -1,23 +1,23 @@ -#pragma once - -#include "defs.h" - -#define LOG_T(stream) LOG_TRACE_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_D(stream) LOG_DEBUG_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_E(stream) LOG_ERROR_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) -#define LOG_C(stream) LOG_CRIT_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) - -namespace NKikimr { +#pragma once + +#include "defs.h" + +#define LOG_T(stream) LOG_TRACE_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_D(stream) LOG_DEBUG_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_E(stream) LOG_ERROR_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) +#define LOG_C(stream) LOG_CRIT_S (*TlsActivationContext, NKikimrServices::CHANGE_EXCHANGE, GetLogPrefix() << stream) + +namespace NKikimr { namespace NDataShard { - -struct TDataShardId { - ui64 TabletId; - ui64 Generation; - TActorId ActorId; -}; - + +struct TDataShardId { + ui64 TabletId; + ui64 Generation; + TActorId ActorId; +}; + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_exchange_impl.h b/ydb/core/tx/datashard/change_exchange_impl.h index 61059896a0c..b6af2aea263 100644 --- a/ydb/core/tx/datashard/change_exchange_impl.h +++ b/ydb/core/tx/datashard/change_exchange_impl.h @@ -1,13 +1,13 @@ -#pragma once - -#include "defs.h" -#include "change_exchange_helpers.h" - -namespace NKikimr { +#pragma once + +#include "defs.h" +#include "change_exchange_helpers.h" + +namespace NKikimr { namespace NDataShard { - -IActor* CreateAsyncIndexChangeSender(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId); -IActor* CreateCdcStreamChangeSender(const TDataShardId& dataShard, const TPathId& streamPathId); - + +IActor* CreateAsyncIndexChangeSender(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId); +IActor* CreateCdcStreamChangeSender(const TDataShardId& dataShard, const TPathId& streamPathId); + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_exchange_split.cpp b/ydb/core/tx/datashard/change_exchange_split.cpp index bf592f6e410..af95c80036b 100644 --- a/ydb/core/tx/datashard/change_exchange_split.cpp +++ b/ydb/core/tx/datashard/change_exchange_split.cpp @@ -1,584 +1,584 @@ -#include "change_exchange.h" -#include "change_exchange_helpers.h" -#include "change_sender_common_ops.h" -#include "datashard_impl.h" - +#include "change_exchange.h" +#include "change_exchange_helpers.h" +#include "change_sender_common_ops.h" +#include "datashard_impl.h" + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/persqueue/events/global.h> #include <ydb/core/persqueue/writer/source_id_encoding.h> #include <ydb/public/lib/base/msgbus_status.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - -#include <util/generic/hash.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NDataShard { - -class TCdcPartitionWorker: public TActorBootstrapped<TCdcPartitionWorker> { - void Ack() { - Send(Parent, new TEvChangeExchange::TEvSplitAck()); - PassAway(); - } - - void Leave() { - Send(Parent, new TEvents::TEvGone()); - PassAway(); - } - - void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { - const auto& response = ev->Get()->Record; - - switch (response.GetStatus()) { - case NMsgBusProxy::MSTATUS_OK: - if (response.GetErrorCode() == NPersQueue::NErrorCode::OK) { - return Ack(); - } - break; - case NMsgBusProxy::MSTATUS_ERROR: - if (response.GetErrorCode() == NPersQueue::NErrorCode::SOURCEID_DELETED) { - return Ack(); - } - break; - default: - break; - } - - Leave(); - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - if (ev->Get()->TabletId == TabletId && ev->Get()->Status != NKikimrProto::OK) { - Leave(); - } - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { - if (ev->Get()->TabletId == TabletId) { - Leave(); - } - } - - void PassAway() override { - if (PipeClient) { - NTabletPipe::CloseAndForgetClient(SelfId(), PipeClient); - } - - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; - } - - explicit TCdcPartitionWorker(const TActorId& parent, ui32 partitionId, ui64 tabletId, - ui64 srcTabletId, const TVector<ui64>& dstTabletIds) - : Parent(parent) - , PartitionId(partitionId) - , TabletId(tabletId) - , SrcTabletId(srcTabletId) - , DstTabletIds(dstTabletIds) - { - } - - void Bootstrap() { - NTabletPipe::TClientConfig config; - config.RetryPolicy = { - .RetryLimitCount = 6, - .MinRetryTime = TDuration::MilliSeconds(10), - .MaxRetryTime = TDuration::MilliSeconds(100), - .BackoffMultiplier = 2, - .DoFirstRetryInstantly = true - }; - - PipeClient = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), TabletId, config)); - - auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); - auto& request = *ev->Record.MutablePartitionRequest(); - request.SetPartition(PartitionId); - ActorIdToProto(PipeClient, request.MutablePipeClient()); - - auto& cmd = *request.MutableCmdSplitMessageGroup(); - { - auto& group = *cmd.AddDeregisterGroups(); - group.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(SrcTabletId))); - } - for (const auto dstTabletId : DstTabletIds) { - auto& group = *cmd.AddRegisterGroups(); - group.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(dstTabletId))); - } - - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPersQueue::TEvResponse, Handle); - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const ui32 PartitionId; - const ui64 TabletId; - const ui64 SrcTabletId; - const TVector<ui64> DstTabletIds; - - TActorId PipeClient; - -}; // TCdcPartitionWorker - -class TCdcWorker: public TActorBootstrapped<TCdcWorker>, private TSchemeCacheHelpers { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[ChangeExchangeSplitCdcWorker]" - << "[" << SrcTabletId << "]" - << SelfId() /* contains brackets */; - } - - return LogPrefix.GetRef(); - } - - void Ack() { - Send(Parent, new TEvChangeExchange::TEvSplitAck()); - PassAway(); - } - - bool IsResolvingCdcStream() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveCdcStream); - } - - bool IsResolvingTopic() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveTopic); - } - - bool IsResolving() const { - return IsResolvingCdcStream() - || IsResolvingTopic(); - } - - TStringBuf CurrentStateName() const { - if (IsResolvingCdcStream()) { - return "ResolveCdcStream"; - } else if (IsResolvingTopic()) { - return "ResolveTopic"; - } else { - return ""; - } - } - - void Retry() { - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); - } - - void LogCritAndRetry(const TString& error) { - LOG_C(error); - Retry(); - } - - void LogWarnAndRetry(const TString& error) { - LOG_W(error); - Retry(); - } - - template <typename CheckFunc, typename FailFunc, typename T, typename... Args> - bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { - return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); - } - - bool CheckNotEmpty(const TAutoPtr<TNavigate>& result) { - return Check(&TSchemeCacheHelpers::CheckNotEmpty<TNavigate>, &TThis::LogCritAndRetry, result); - } - - bool CheckEntriesCount(const TAutoPtr<TNavigate>& result, ui32 expected) { - return Check(&TSchemeCacheHelpers::CheckEntriesCount<TNavigate>, &TThis::LogCritAndRetry, result, expected); - } - - bool CheckTableId(const TNavigate::TEntry& entry, const TTableId& expected) { - return Check(&TSchemeCacheHelpers::CheckTableId<TNavigate::TEntry>, &TThis::LogCritAndRetry, entry, expected); - } - - bool CheckEntrySucceeded(const TNavigate::TEntry& entry) { - return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry); - } - - bool CheckEntryKind(const TNavigate::TEntry& entry, TNavigate::EKind expected) { - return Check(&TSchemeCacheHelpers::CheckEntryKind<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry, expected); - } - - bool CheckNotEmpty(const TIntrusiveConstPtr<TNavigate::TPQGroupInfo>& pqInfo) { - if (pqInfo) { - return true; - } - - LogCritAndRetry(TStringBuilder() << "Empty pq info at '" << CurrentStateName() << "'"); - return false; - } - - /// ResolveCdcStream - - void ResolveCdcStream() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveCdcStream); - } - - STATEFN(StateResolveCdcStream) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleCdcStream); - sFunc(TEvents::TEvWakeup, ResolveCdcStream); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleCdcStream(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, PathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindCdcStream)) { - return; - } - - Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); - const auto& topic = entry.ListNodeEntry->Children.at(0); - - Y_VERIFY(topic.Kind == TNavigate::KindTopic); - ResolveTopic(topic.PathId); - } - - /// ResolveTopic - - void ResolveTopic(const TPathId& pathId) { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(pathId, TNavigate::OpTopic)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveTopic); - } - - STATEFN(StateResolveTopic) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTopic); - sFunc(TEvents::TEvWakeup, ResolveCdcStream); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindTopic)) { - return; - } - - if (!CheckNotEmpty(entry.PQGroupInfo)) { - return; - } - - THashMap<ui32, TActorId> workers; - - for (const auto& partition : entry.PQGroupInfo->Description.GetPartitions()) { - const auto partitionId = partition.GetPartitionId(); - const auto tabletId = partition.GetTabletId(); - - auto it = Workers.find(partitionId); - if (it != Workers.end()) { - workers.emplace(partitionId, it->second); - Workers.erase(it); - } else { - const auto worker = Register(new TCdcPartitionWorker(SelfId(), partitionId, tabletId, SrcTabletId, DstTabletIds)); - workers.emplace(partitionId, worker); - Pending.emplace(worker, partitionId); - } - } - - for (const auto& [_, worker] : Workers) { - if (worker) { - Send(worker, new TEvents::TEvPoisonPill()); - Pending.erase(worker); - } - } - - if (Pending.empty()) { - return Ack(); - } - - Workers = std::move(workers); - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvSplitAck, Handle); - hFunc(TEvents::TEvGone, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - auto it = Pending.find(ev->Sender); - if (it == Pending.end()) { - LOG_W("Split ack from unknown worker" - << ": worker# " << ev->Sender); - return; - } - - LOG_N("Split ack" - << ": worker# " << it->first - << ", partition# " << it->second); - - Workers[it->second] = TActorId(); - Pending.erase(it); - - if (Pending.empty()) { - Ack(); - } - } - - void Handle(TEvents::TEvGone::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - auto it = Pending.find(ev->Sender); - if (it == Pending.end()) { - LOG_W("Unknown worker has gone" - << ": worker# " << ev->Sender); - return; - } - - LOG_I("Worker has gone" - << ": worker# " << it->first - << ", partition# " << it->second); - - Workers.erase(it->second); - Pending.erase(it); - - if (!IsResolving()) { - ResolveCdcStream(); - } - } - - void PassAway() override { - for (const auto& [_, worker] : Workers) { - if (worker) { - Send(worker, new TEvents::TEvPoisonPill()); - } - } - - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; - } - - explicit TCdcWorker(const TActorId& parent, const TPathId& pathId, - ui64 srcTabletId, const TVector<ui64>& dstTabletIds) - : Parent(parent) - , PathId(pathId) - , SrcTabletId(srcTabletId) - , DstTabletIds(dstTabletIds) - { - } - - void Bootstrap() { - ResolveCdcStream(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const TPathId PathId; - const ui64 SrcTabletId; - const TVector<ui64> DstTabletIds; - mutable TMaybe<TString> LogPrefix; - - THashMap<ui32, TActorId> Workers; - THashMap<TActorId, ui32> Pending; - -}; // TCdcWorker - -class TChangeExchageSplit: public TActorBootstrapped<TChangeExchageSplit> { - using EWorkerType = TEvChangeExchange::ESenderType; - - struct TWorker { - EWorkerType Type; - TActorId ActorId; - - explicit TWorker(EWorkerType type) - : Type(type) - { - } - }; - - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[ChangeExchangeSplit]" - << "[" << DataShard.TabletId << "]" - << SelfId() /* contains brackets */; - } - - return LogPrefix.GetRef(); - } - - TActorId RegisterWorker(const TPathId& pathId, EWorkerType type) const { - switch (type) { - case EWorkerType::CdcStream: - return Register(new TCdcWorker(SelfId(), pathId, DataShard.TabletId, DstDataShards)); - case EWorkerType::AsyncIndex: - Y_FAIL("unreachable"); - } - } - - TActorId RegisterWorker(const TPathId& pathId, TWorker& worker) const { - Y_VERIFY_DEBUG(!worker.ActorId); - worker.ActorId = RegisterWorker(pathId, worker.Type); - return worker.ActorId; - } - - void Ack() { - Send(DataShard.ActorId, new TEvChangeExchange::TEvSplitAck()); - PassAway(); - } - - void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - auto it = Pending.find(ev->Sender); - if (it == Pending.end()) { - LOG_W("Split ack from unknown worker" - << ": worker# " << ev->Sender); - return; - } - - LOG_N("Split ack" - << ": worker# " << ev->Sender); - - Workers.at(it->second).ActorId = TActorId(); - Pending.erase(it); - - if (Pending.empty()) { - Ack(); - } - } - - void PassAway() override { - for (const auto& [_, worker] : Workers) { - if (worker.ActorId) { - Send(worker.ActorId, new TEvents::TEvPoisonPill()); - } - } - - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; - } - - explicit TChangeExchageSplit(const TDataShard* self, const TVector<ui64>& dstDataShards) - : TActorBootstrapped() - , DataShard{self->TabletID(), self->Generation(), self->SelfId()} - , DstDataShards(dstDataShards) - { - for (const auto& [_, tableInfo] : self->GetUserTables()) { - for (const auto& [streamPathId, _] : tableInfo->CdcStreams) { - Workers.emplace(streamPathId, EWorkerType::CdcStream); - } - } - } - - void Bootstrap() { - if (!Workers) { - return Ack(); - } - - for (auto& [pathId, worker] : Workers) { - Pending.emplace(RegisterWorker(pathId, worker), pathId); - } - - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvSplitAck, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TDataShardId DataShard; - const TVector<ui64> DstDataShards; - mutable TMaybe<TString> LogPrefix; - - THashMap<TPathId, TWorker> Workers; - THashMap<TActorId, TPathId> Pending; - -}; // TChangeExchageSplit - -IActor* CreateChangeExchangeSplit(const TDataShard* self, const TVector<ui64>& dstDataShards) { - return new TChangeExchageSplit(self, dstDataShards); -} - -} // NDataShard -} // NKikimr + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/generic/hash.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NDataShard { + +class TCdcPartitionWorker: public TActorBootstrapped<TCdcPartitionWorker> { + void Ack() { + Send(Parent, new TEvChangeExchange::TEvSplitAck()); + PassAway(); + } + + void Leave() { + Send(Parent, new TEvents::TEvGone()); + PassAway(); + } + + void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { + const auto& response = ev->Get()->Record; + + switch (response.GetStatus()) { + case NMsgBusProxy::MSTATUS_OK: + if (response.GetErrorCode() == NPersQueue::NErrorCode::OK) { + return Ack(); + } + break; + case NMsgBusProxy::MSTATUS_ERROR: + if (response.GetErrorCode() == NPersQueue::NErrorCode::SOURCEID_DELETED) { + return Ack(); + } + break; + default: + break; + } + + Leave(); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->TabletId == TabletId && ev->Get()->Status != NKikimrProto::OK) { + Leave(); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { + if (ev->Get()->TabletId == TabletId) { + Leave(); + } + } + + void PassAway() override { + if (PipeClient) { + NTabletPipe::CloseAndForgetClient(SelfId(), PipeClient); + } + + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; + } + + explicit TCdcPartitionWorker(const TActorId& parent, ui32 partitionId, ui64 tabletId, + ui64 srcTabletId, const TVector<ui64>& dstTabletIds) + : Parent(parent) + , PartitionId(partitionId) + , TabletId(tabletId) + , SrcTabletId(srcTabletId) + , DstTabletIds(dstTabletIds) + { + } + + void Bootstrap() { + NTabletPipe::TClientConfig config; + config.RetryPolicy = { + .RetryLimitCount = 6, + .MinRetryTime = TDuration::MilliSeconds(10), + .MaxRetryTime = TDuration::MilliSeconds(100), + .BackoffMultiplier = 2, + .DoFirstRetryInstantly = true + }; + + PipeClient = RegisterWithSameMailbox(NTabletPipe::CreateClient(SelfId(), TabletId, config)); + + auto ev = MakeHolder<TEvPersQueue::TEvRequest>(); + auto& request = *ev->Record.MutablePartitionRequest(); + request.SetPartition(PartitionId); + ActorIdToProto(PipeClient, request.MutablePipeClient()); + + auto& cmd = *request.MutableCmdSplitMessageGroup(); + { + auto& group = *cmd.AddDeregisterGroups(); + group.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(SrcTabletId))); + } + for (const auto dstTabletId : DstTabletIds) { + auto& group = *cmd.AddRegisterGroups(); + group.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(dstTabletId))); + } + + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPersQueue::TEvResponse, Handle); + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const ui32 PartitionId; + const ui64 TabletId; + const ui64 SrcTabletId; + const TVector<ui64> DstTabletIds; + + TActorId PipeClient; + +}; // TCdcPartitionWorker + +class TCdcWorker: public TActorBootstrapped<TCdcWorker>, private TSchemeCacheHelpers { + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[ChangeExchangeSplitCdcWorker]" + << "[" << SrcTabletId << "]" + << SelfId() /* contains brackets */; + } + + return LogPrefix.GetRef(); + } + + void Ack() { + Send(Parent, new TEvChangeExchange::TEvSplitAck()); + PassAway(); + } + + bool IsResolvingCdcStream() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveCdcStream); + } + + bool IsResolvingTopic() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveTopic); + } + + bool IsResolving() const { + return IsResolvingCdcStream() + || IsResolvingTopic(); + } + + TStringBuf CurrentStateName() const { + if (IsResolvingCdcStream()) { + return "ResolveCdcStream"; + } else if (IsResolvingTopic()) { + return "ResolveTopic"; + } else { + return ""; + } + } + + void Retry() { + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); + } + + void LogCritAndRetry(const TString& error) { + LOG_C(error); + Retry(); + } + + void LogWarnAndRetry(const TString& error) { + LOG_W(error); + Retry(); + } + + template <typename CheckFunc, typename FailFunc, typename T, typename... Args> + bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { + return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); + } + + bool CheckNotEmpty(const TAutoPtr<TNavigate>& result) { + return Check(&TSchemeCacheHelpers::CheckNotEmpty<TNavigate>, &TThis::LogCritAndRetry, result); + } + + bool CheckEntriesCount(const TAutoPtr<TNavigate>& result, ui32 expected) { + return Check(&TSchemeCacheHelpers::CheckEntriesCount<TNavigate>, &TThis::LogCritAndRetry, result, expected); + } + + bool CheckTableId(const TNavigate::TEntry& entry, const TTableId& expected) { + return Check(&TSchemeCacheHelpers::CheckTableId<TNavigate::TEntry>, &TThis::LogCritAndRetry, entry, expected); + } + + bool CheckEntrySucceeded(const TNavigate::TEntry& entry) { + return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry); + } + + bool CheckEntryKind(const TNavigate::TEntry& entry, TNavigate::EKind expected) { + return Check(&TSchemeCacheHelpers::CheckEntryKind<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry, expected); + } + + bool CheckNotEmpty(const TIntrusiveConstPtr<TNavigate::TPQGroupInfo>& pqInfo) { + if (pqInfo) { + return true; + } + + LogCritAndRetry(TStringBuilder() << "Empty pq info at '" << CurrentStateName() << "'"); + return false; + } + + /// ResolveCdcStream + + void ResolveCdcStream() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveCdcStream); + } + + STATEFN(StateResolveCdcStream) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleCdcStream); + sFunc(TEvents::TEvWakeup, ResolveCdcStream); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleCdcStream(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, PathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindCdcStream)) { + return; + } + + Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); + const auto& topic = entry.ListNodeEntry->Children.at(0); + + Y_VERIFY(topic.Kind == TNavigate::KindTopic); + ResolveTopic(topic.PathId); + } + + /// ResolveTopic + + void ResolveTopic(const TPathId& pathId) { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(pathId, TNavigate::OpTopic)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveTopic); + } + + STATEFN(StateResolveTopic) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTopic); + sFunc(TEvents::TEvWakeup, ResolveCdcStream); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindTopic)) { + return; + } + + if (!CheckNotEmpty(entry.PQGroupInfo)) { + return; + } + + THashMap<ui32, TActorId> workers; + + for (const auto& partition : entry.PQGroupInfo->Description.GetPartitions()) { + const auto partitionId = partition.GetPartitionId(); + const auto tabletId = partition.GetTabletId(); + + auto it = Workers.find(partitionId); + if (it != Workers.end()) { + workers.emplace(partitionId, it->second); + Workers.erase(it); + } else { + const auto worker = Register(new TCdcPartitionWorker(SelfId(), partitionId, tabletId, SrcTabletId, DstTabletIds)); + workers.emplace(partitionId, worker); + Pending.emplace(worker, partitionId); + } + } + + for (const auto& [_, worker] : Workers) { + if (worker) { + Send(worker, new TEvents::TEvPoisonPill()); + Pending.erase(worker); + } + } + + if (Pending.empty()) { + return Ack(); + } + + Workers = std::move(workers); + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvSplitAck, Handle); + hFunc(TEvents::TEvGone, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + auto it = Pending.find(ev->Sender); + if (it == Pending.end()) { + LOG_W("Split ack from unknown worker" + << ": worker# " << ev->Sender); + return; + } + + LOG_N("Split ack" + << ": worker# " << it->first + << ", partition# " << it->second); + + Workers[it->second] = TActorId(); + Pending.erase(it); + + if (Pending.empty()) { + Ack(); + } + } + + void Handle(TEvents::TEvGone::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + auto it = Pending.find(ev->Sender); + if (it == Pending.end()) { + LOG_W("Unknown worker has gone" + << ": worker# " << ev->Sender); + return; + } + + LOG_I("Worker has gone" + << ": worker# " << it->first + << ", partition# " << it->second); + + Workers.erase(it->second); + Pending.erase(it); + + if (!IsResolving()) { + ResolveCdcStream(); + } + } + + void PassAway() override { + for (const auto& [_, worker] : Workers) { + if (worker) { + Send(worker, new TEvents::TEvPoisonPill()); + } + } + + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; + } + + explicit TCdcWorker(const TActorId& parent, const TPathId& pathId, + ui64 srcTabletId, const TVector<ui64>& dstTabletIds) + : Parent(parent) + , PathId(pathId) + , SrcTabletId(srcTabletId) + , DstTabletIds(dstTabletIds) + { + } + + void Bootstrap() { + ResolveCdcStream(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const TPathId PathId; + const ui64 SrcTabletId; + const TVector<ui64> DstTabletIds; + mutable TMaybe<TString> LogPrefix; + + THashMap<ui32, TActorId> Workers; + THashMap<TActorId, ui32> Pending; + +}; // TCdcWorker + +class TChangeExchageSplit: public TActorBootstrapped<TChangeExchageSplit> { + using EWorkerType = TEvChangeExchange::ESenderType; + + struct TWorker { + EWorkerType Type; + TActorId ActorId; + + explicit TWorker(EWorkerType type) + : Type(type) + { + } + }; + + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[ChangeExchangeSplit]" + << "[" << DataShard.TabletId << "]" + << SelfId() /* contains brackets */; + } + + return LogPrefix.GetRef(); + } + + TActorId RegisterWorker(const TPathId& pathId, EWorkerType type) const { + switch (type) { + case EWorkerType::CdcStream: + return Register(new TCdcWorker(SelfId(), pathId, DataShard.TabletId, DstDataShards)); + case EWorkerType::AsyncIndex: + Y_FAIL("unreachable"); + } + } + + TActorId RegisterWorker(const TPathId& pathId, TWorker& worker) const { + Y_VERIFY_DEBUG(!worker.ActorId); + worker.ActorId = RegisterWorker(pathId, worker.Type); + return worker.ActorId; + } + + void Ack() { + Send(DataShard.ActorId, new TEvChangeExchange::TEvSplitAck()); + PassAway(); + } + + void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + auto it = Pending.find(ev->Sender); + if (it == Pending.end()) { + LOG_W("Split ack from unknown worker" + << ": worker# " << ev->Sender); + return; + } + + LOG_N("Split ack" + << ": worker# " << ev->Sender); + + Workers.at(it->second).ActorId = TActorId(); + Pending.erase(it); + + if (Pending.empty()) { + Ack(); + } + } + + void PassAway() override { + for (const auto& [_, worker] : Workers) { + if (worker.ActorId) { + Send(worker.ActorId, new TEvents::TEvPoisonPill()); + } + } + + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_EXCHANGE_SPLIT_ACTOR; + } + + explicit TChangeExchageSplit(const TDataShard* self, const TVector<ui64>& dstDataShards) + : TActorBootstrapped() + , DataShard{self->TabletID(), self->Generation(), self->SelfId()} + , DstDataShards(dstDataShards) + { + for (const auto& [_, tableInfo] : self->GetUserTables()) { + for (const auto& [streamPathId, _] : tableInfo->CdcStreams) { + Workers.emplace(streamPathId, EWorkerType::CdcStream); + } + } + } + + void Bootstrap() { + if (!Workers) { + return Ack(); + } + + for (auto& [pathId, worker] : Workers) { + Pending.emplace(RegisterWorker(pathId, worker), pathId); + } + + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvSplitAck, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TDataShardId DataShard; + const TVector<ui64> DstDataShards; + mutable TMaybe<TString> LogPrefix; + + THashMap<TPathId, TWorker> Workers; + THashMap<TActorId, TPathId> Pending; + +}; // TChangeExchageSplit + +IActor* CreateChangeExchangeSplit(const TDataShard* self, const TVector<ui64>& dstDataShards) { + return new TChangeExchageSplit(self, dstDataShards); +} + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/change_record.cpp b/ydb/core/tx/datashard/change_record.cpp index 655ea38ffbb..661df31fb00 100644 --- a/ydb/core/tx/datashard/change_record.cpp +++ b/ydb/core/tx/datashard/change_record.cpp @@ -1,121 +1,121 @@ -#include "change_record.h" - +#include "change_record.h" + #include <ydb/core/protos/change_exchange.pb.h> - -#include <util/stream/str.h> - -namespace NKikimr { + +#include <util/stream/str.h> + +namespace NKikimr { namespace NDataShard { - -i64 TChangeRecord::GetSeqNo() const { - Y_VERIFY(Order <= Max<i64>()); - return static_cast<i64>(Order); -} - -TConstArrayRef<TCell> TChangeRecord::GetKey() const { - if (Key) { - return *Key; - } - - switch (Kind) { - case EKind::AsyncIndex: - case EKind::CdcDataChange: { - NKikimrChangeExchange::TChangeRecord::TDataChange parsed; - Y_VERIFY(parsed.ParseFromArray(Body.data(), Body.size())); - - TSerializedCellVec key; - Y_VERIFY(TSerializedCellVec::TryParse(parsed.GetKey().GetData(), key)); - - Key.ConstructInPlace(key.GetCells()); - break; - } - } - - Y_VERIFY(Key); - return *Key; -} - -void TChangeRecord::SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const { - record.SetOrder(Order); - record.SetGroup(Group); - record.SetStep(Step); - record.SetTxId(TxId); - record.SetPathOwnerId(PathId.OwnerId); - record.SetLocalPathId(PathId.LocalPathId); - - switch (Kind) { - case EKind::AsyncIndex: { - Y_VERIFY(record.MutableAsyncIndex()->ParseFromArray(Body.data(), Body.size())); - break; - } - case EKind::CdcDataChange: { - Y_VERIFY(record.MutableCdcDataChange()->ParseFromArray(Body.data(), Body.size())); - break; - } - } -} - -TString TChangeRecord::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TChangeRecord::Out(IOutputStream& out) const { - out << "{" - << " Order: " << Order - << " Group: " << Group - << " Step: " << Step - << " TxId: " << TxId - << " PathId: " << PathId - << " Kind: " << Kind - << " Body: " << Body.size() << "b" - << " }"; -} - -TChangeRecordBuilder::TChangeRecordBuilder(EKind kind) { - Record.Kind = kind; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithOrder(ui64 order) { - Record.Order = order; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithGroup(ui64 group) { - Record.Group = group; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithStep(ui64 step) { - Record.Step = step; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithTxId(ui64 txId) { - Record.TxId = txId; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithPathId(const TPathId& pathId) { - Record.PathId = pathId; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithBody(const TString& body) { - Record.Body = body; - return *this; -} - -TChangeRecordBuilder& TChangeRecordBuilder::WithBody(TString&& body) { - Record.Body = std::move(body); - return *this; -} - -TChangeRecord&& TChangeRecordBuilder::Build() { - return std::move(Record); -} - + +i64 TChangeRecord::GetSeqNo() const { + Y_VERIFY(Order <= Max<i64>()); + return static_cast<i64>(Order); +} + +TConstArrayRef<TCell> TChangeRecord::GetKey() const { + if (Key) { + return *Key; + } + + switch (Kind) { + case EKind::AsyncIndex: + case EKind::CdcDataChange: { + NKikimrChangeExchange::TChangeRecord::TDataChange parsed; + Y_VERIFY(parsed.ParseFromArray(Body.data(), Body.size())); + + TSerializedCellVec key; + Y_VERIFY(TSerializedCellVec::TryParse(parsed.GetKey().GetData(), key)); + + Key.ConstructInPlace(key.GetCells()); + break; + } + } + + Y_VERIFY(Key); + return *Key; +} + +void TChangeRecord::SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const { + record.SetOrder(Order); + record.SetGroup(Group); + record.SetStep(Step); + record.SetTxId(TxId); + record.SetPathOwnerId(PathId.OwnerId); + record.SetLocalPathId(PathId.LocalPathId); + + switch (Kind) { + case EKind::AsyncIndex: { + Y_VERIFY(record.MutableAsyncIndex()->ParseFromArray(Body.data(), Body.size())); + break; + } + case EKind::CdcDataChange: { + Y_VERIFY(record.MutableCdcDataChange()->ParseFromArray(Body.data(), Body.size())); + break; + } + } +} + +TString TChangeRecord::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TChangeRecord::Out(IOutputStream& out) const { + out << "{" + << " Order: " << Order + << " Group: " << Group + << " Step: " << Step + << " TxId: " << TxId + << " PathId: " << PathId + << " Kind: " << Kind + << " Body: " << Body.size() << "b" + << " }"; +} + +TChangeRecordBuilder::TChangeRecordBuilder(EKind kind) { + Record.Kind = kind; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithOrder(ui64 order) { + Record.Order = order; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithGroup(ui64 group) { + Record.Group = group; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithStep(ui64 step) { + Record.Step = step; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithTxId(ui64 txId) { + Record.TxId = txId; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithPathId(const TPathId& pathId) { + Record.PathId = pathId; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithBody(const TString& body) { + Record.Body = body; + return *this; +} + +TChangeRecordBuilder& TChangeRecordBuilder::WithBody(TString&& body) { + Record.Body = std::move(body); + return *this; +} + +TChangeRecord&& TChangeRecordBuilder::Build() { + return std::move(Record); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_record.h b/ydb/core/tx/datashard/change_record.h index e51d6390a3e..7c88bb8b663 100644 --- a/ydb/core/tx/datashard/change_record.h +++ b/ydb/core/tx/datashard/change_record.h @@ -1,83 +1,83 @@ -#pragma once - +#pragma once + #include <ydb/core/base/pathid.h> #include <ydb/core/scheme/scheme_tablecell.h> - -#include <util/generic/maybe.h> -#include <util/generic/string.h> - -namespace NKikimrChangeExchange { - class TChangeRecord; -} - -namespace NKikimr { + +#include <util/generic/maybe.h> +#include <util/generic/string.h> + +namespace NKikimrChangeExchange { + class TChangeRecord; +} + +namespace NKikimr { namespace NDataShard { - -class TChangeRecordBuilder; - -class TChangeRecord { - friend class TChangeRecordBuilder; - -public: - enum class EKind: ui8 { - AsyncIndex, - CdcDataChange, - }; - -public: - ui64 GetOrder() const { return Order; } - ui64 GetGroup() const { return Group; } - ui64 GetStep() const { return Step; } - ui64 GetTxId() const { return TxId; } - const TPathId& GetPathId() const { return PathId; } - EKind GetKind() const { return Kind; } - const TString& GetBody() const { return Body; } - i64 GetSeqNo() const; - TConstArrayRef<TCell> GetKey() const; - - void SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const; - - TString ToString() const; - void Out(IOutputStream& out) const; - -private: - ui64 Order; - ui64 Group; - ui64 Step; - ui64 TxId; - TPathId PathId; - EKind Kind; - TString Body; - - mutable TMaybe<TOwnedCellVec> Key; - -}; // TChangeRecord - -class TChangeRecordBuilder { - using EKind = TChangeRecord::EKind; - -public: - explicit TChangeRecordBuilder(EKind kind); - - TChangeRecordBuilder& WithOrder(ui64 order); - TChangeRecordBuilder& WithGroup(ui64 group); - TChangeRecordBuilder& WithStep(ui64 step); - TChangeRecordBuilder& WithTxId(ui64 txId); - TChangeRecordBuilder& WithPathId(const TPathId& pathId); - - TChangeRecordBuilder& WithBody(const TString& body); - TChangeRecordBuilder& WithBody(TString&& body); - - TChangeRecord&& Build(); - -private: - TChangeRecord Record; - -}; // TChangeRecordBuilder - + +class TChangeRecordBuilder; + +class TChangeRecord { + friend class TChangeRecordBuilder; + +public: + enum class EKind: ui8 { + AsyncIndex, + CdcDataChange, + }; + +public: + ui64 GetOrder() const { return Order; } + ui64 GetGroup() const { return Group; } + ui64 GetStep() const { return Step; } + ui64 GetTxId() const { return TxId; } + const TPathId& GetPathId() const { return PathId; } + EKind GetKind() const { return Kind; } + const TString& GetBody() const { return Body; } + i64 GetSeqNo() const; + TConstArrayRef<TCell> GetKey() const; + + void SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const; + + TString ToString() const; + void Out(IOutputStream& out) const; + +private: + ui64 Order; + ui64 Group; + ui64 Step; + ui64 TxId; + TPathId PathId; + EKind Kind; + TString Body; + + mutable TMaybe<TOwnedCellVec> Key; + +}; // TChangeRecord + +class TChangeRecordBuilder { + using EKind = TChangeRecord::EKind; + +public: + explicit TChangeRecordBuilder(EKind kind); + + TChangeRecordBuilder& WithOrder(ui64 order); + TChangeRecordBuilder& WithGroup(ui64 group); + TChangeRecordBuilder& WithStep(ui64 step); + TChangeRecordBuilder& WithTxId(ui64 txId); + TChangeRecordBuilder& WithPathId(const TPathId& pathId); + + TChangeRecordBuilder& WithBody(const TString& body); + TChangeRecordBuilder& WithBody(TString&& body); + + TChangeRecord&& Build(); + +private: + TChangeRecord Record; + +}; // TChangeRecordBuilder + } // NDataShard -} // NKikimr - +} // NKikimr + Y_DECLARE_OUT_SPEC(inline, NKikimr::NDataShard::TChangeRecord, out, value) { - return value.Out(out); -} + return value.Out(out); +} diff --git a/ydb/core/tx/datashard/change_sender.cpp b/ydb/core/tx/datashard/change_sender.cpp index 3300b3035ef..1b7ab294c03 100644 --- a/ydb/core/tx/datashard/change_sender.cpp +++ b/ydb/core/tx/datashard/change_sender.cpp @@ -1,235 +1,235 @@ -#include "change_exchange.h" -#include "change_exchange_impl.h" -#include "datashard_impl.h" - +#include "change_exchange.h" +#include "change_exchange_impl.h" +#include "datashard_impl.h" + #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - -#include <util/generic/hash.h> -#include <util/generic/maybe.h> - -namespace NKikimr { + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/generic/hash.h> +#include <util/generic/maybe.h> + +namespace NKikimr { namespace NDataShard { - -class TChangeSender: public TActor<TChangeSender> { - using ESenderType = TEvChangeExchange::ESenderType; - using TEnqueuedRecord = TEvChangeExchange::TEvEnqueueRecords::TRecordInfo; - - struct TSender { - TTableId UserTableId; - ESenderType Type; - TActorId ActorId; - }; - - bool IsActive() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateActive); - } - - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[ChangeSender]" - << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" - << SelfId() /* contains brackets */ - << (IsActive() ? "" : "[Inactive]") << " "; - } - - return LogPrefix.GetRef(); - } - - TSender& AddChangeSender(const TPathId& pathId, const TTableId& userTableId, ESenderType type) { - Y_VERIFY_DEBUG(!Senders.contains(pathId)); - - auto& sender = Senders[pathId]; - sender.UserTableId = userTableId; - sender.Type = type; - - return sender; - } - - TActorId RegisterChangeSender(const TPathId& pathId, const TTableId& userTableId, ESenderType type) const { - switch (type) { - case ESenderType::AsyncIndex: - return Register(CreateAsyncIndexChangeSender(DataShard, userTableId, pathId)); - case ESenderType::CdcStream: - return Register(CreateCdcStreamChangeSender(DataShard, pathId)); - } - } - - void RegisterChangeSender(const TPathId& pathId, TSender& sender) const { - Y_VERIFY_DEBUG(!sender.ActorId); - sender.ActorId = RegisterChangeSender(pathId, sender.UserTableId, sender.Type); - } - - void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - auto& records = ev->Get()->Records; - - if (!IsActive()) { - std::move(records.begin(), records.end(), std::back_inserter(Enqueued)); - return; - } - - THashMap<TActorId, TVector<TEnqueuedRecord>> forward; - TVector<ui64> remove; - - for (auto& record : records) { - auto it = Senders.find(record.PathId); - if (it != Senders.end()) { - forward[it->second.ActorId].push_back(std::move(record)); - } else { - remove.push_back(record.Order); - } - } - - for (auto& [to, records] : forward) { - Send(to, new TEvChangeExchange::TEvEnqueueRecords(std::move(records))); - } - - if (remove) { - Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); - } - } - - void Handle(TEvChangeExchange::TEvAddSender::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - const auto& msg = *ev->Get(); - - auto it = Senders.find(msg.PathId); - if (it != Senders.end()) { - Y_VERIFY(it->second.UserTableId == msg.UserTableId); - Y_VERIFY(it->second.Type == msg.Type); - LOG_W("Trying to add duplicate sender" - << ": userTableId# " << msg.UserTableId - << ", type# " << msg.Type - << ", pathId# " << msg.PathId); - return; - } - - LOG_N("Add sender" - << ": userTableId# " << msg.UserTableId - << ", type# " << msg.Type - << ", pathId# " << msg.PathId); - - auto& sender = AddChangeSender(msg.PathId, msg.UserTableId, msg.Type); - if (IsActive()) { - RegisterChangeSender(msg.PathId, sender); - } - } - - void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - const auto& pathId = ev->Get()->PathId; - - auto it = Senders.find(pathId); - if (it == Senders.end()) { - LOG_W("Trying to remove unknown sender" - << ": pathId# " << pathId); - return; - } - - LOG_N("Remove sender" - << ": type# " << it->second.Type - << ", pathId# " << it->first); - - if (const auto& actorId = it->second.ActorId) { - Send(actorId, new TEvChangeExchange::TEvRemoveSender(pathId)); - } - - Senders.erase(it); - } - - void Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - Become(&TThis::StateActive); - LogPrefix.Clear(); - - for (auto& [pathId, sender] : Senders) { - RegisterChangeSender(pathId, sender); - } - - if (Enqueued) { - Send(SelfId(), new TEvChangeExchange::TEvEnqueueRecords(std::move(Enqueued))); - } - } - - void PassAway() override { - for (const auto& [_, sender] : Senders) { - if (!sender.ActorId) { - continue; - } - - Send(sender.ActorId, new TEvents::TEvPoisonPill()); - } - - TActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_SENDER_ACTOR; - } - + +class TChangeSender: public TActor<TChangeSender> { + using ESenderType = TEvChangeExchange::ESenderType; + using TEnqueuedRecord = TEvChangeExchange::TEvEnqueueRecords::TRecordInfo; + + struct TSender { + TTableId UserTableId; + ESenderType Type; + TActorId ActorId; + }; + + bool IsActive() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateActive); + } + + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[ChangeSender]" + << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" + << SelfId() /* contains brackets */ + << (IsActive() ? "" : "[Inactive]") << " "; + } + + return LogPrefix.GetRef(); + } + + TSender& AddChangeSender(const TPathId& pathId, const TTableId& userTableId, ESenderType type) { + Y_VERIFY_DEBUG(!Senders.contains(pathId)); + + auto& sender = Senders[pathId]; + sender.UserTableId = userTableId; + sender.Type = type; + + return sender; + } + + TActorId RegisterChangeSender(const TPathId& pathId, const TTableId& userTableId, ESenderType type) const { + switch (type) { + case ESenderType::AsyncIndex: + return Register(CreateAsyncIndexChangeSender(DataShard, userTableId, pathId)); + case ESenderType::CdcStream: + return Register(CreateCdcStreamChangeSender(DataShard, pathId)); + } + } + + void RegisterChangeSender(const TPathId& pathId, TSender& sender) const { + Y_VERIFY_DEBUG(!sender.ActorId); + sender.ActorId = RegisterChangeSender(pathId, sender.UserTableId, sender.Type); + } + + void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + auto& records = ev->Get()->Records; + + if (!IsActive()) { + std::move(records.begin(), records.end(), std::back_inserter(Enqueued)); + return; + } + + THashMap<TActorId, TVector<TEnqueuedRecord>> forward; + TVector<ui64> remove; + + for (auto& record : records) { + auto it = Senders.find(record.PathId); + if (it != Senders.end()) { + forward[it->second.ActorId].push_back(std::move(record)); + } else { + remove.push_back(record.Order); + } + } + + for (auto& [to, records] : forward) { + Send(to, new TEvChangeExchange::TEvEnqueueRecords(std::move(records))); + } + + if (remove) { + Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); + } + } + + void Handle(TEvChangeExchange::TEvAddSender::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + const auto& msg = *ev->Get(); + + auto it = Senders.find(msg.PathId); + if (it != Senders.end()) { + Y_VERIFY(it->second.UserTableId == msg.UserTableId); + Y_VERIFY(it->second.Type == msg.Type); + LOG_W("Trying to add duplicate sender" + << ": userTableId# " << msg.UserTableId + << ", type# " << msg.Type + << ", pathId# " << msg.PathId); + return; + } + + LOG_N("Add sender" + << ": userTableId# " << msg.UserTableId + << ", type# " << msg.Type + << ", pathId# " << msg.PathId); + + auto& sender = AddChangeSender(msg.PathId, msg.UserTableId, msg.Type); + if (IsActive()) { + RegisterChangeSender(msg.PathId, sender); + } + } + + void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + const auto& pathId = ev->Get()->PathId; + + auto it = Senders.find(pathId); + if (it == Senders.end()) { + LOG_W("Trying to remove unknown sender" + << ": pathId# " << pathId); + return; + } + + LOG_N("Remove sender" + << ": type# " << it->second.Type + << ", pathId# " << it->first); + + if (const auto& actorId = it->second.ActorId) { + Send(actorId, new TEvChangeExchange::TEvRemoveSender(pathId)); + } + + Senders.erase(it); + } + + void Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + Become(&TThis::StateActive); + LogPrefix.Clear(); + + for (auto& [pathId, sender] : Senders) { + RegisterChangeSender(pathId, sender); + } + + if (Enqueued) { + Send(SelfId(), new TEvChangeExchange::TEvEnqueueRecords(std::move(Enqueued))); + } + } + + void PassAway() override { + for (const auto& [_, sender] : Senders) { + if (!sender.ActorId) { + continue; + } + + Send(sender.ActorId, new TEvents::TEvPoisonPill()); + } + + TActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_SENDER_ACTOR; + } + explicit TChangeSender(const TDataShard* self) - : TActor(&TThis::StateInactive) - , DataShard{self->TabletID(), self->Generation(), self->SelfId()} - { - for (const auto& [tableId, tableInfo] : self->GetUserTables()) { - const auto fullTableId = TTableId(self->GetPathOwnerId(), tableId); - - for (const auto& [indexPathId, indexInfo] : tableInfo->Indexes) { - if (indexInfo.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { - continue; - } - - AddChangeSender(indexPathId, fullTableId, ESenderType::AsyncIndex); - } - - for (const auto& [streamPathId, _] : tableInfo->CdcStreams) { - AddChangeSender(streamPathId, fullTableId, ESenderType::CdcStream); - } - } - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); - hFunc(TEvChangeExchange::TEvAddSender, Handle); - hFunc(TEvChangeExchange::TEvRemoveSender, Handle); - - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - - STATEFN(StateInactive) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvActivateSender, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - STATEFN(StateActive) { - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - -private: - const TDataShardId DataShard; - mutable TMaybe<TString> LogPrefix; - - THashMap<TPathId, TSender> Senders; - TVector<TEnqueuedRecord> Enqueued; // Enqueued while inactive - -}; // TChangeSender - + : TActor(&TThis::StateInactive) + , DataShard{self->TabletID(), self->Generation(), self->SelfId()} + { + for (const auto& [tableId, tableInfo] : self->GetUserTables()) { + const auto fullTableId = TTableId(self->GetPathOwnerId(), tableId); + + for (const auto& [indexPathId, indexInfo] : tableInfo->Indexes) { + if (indexInfo.Type != TUserTable::TTableIndex::EIndexType::EIndexTypeGlobalAsync) { + continue; + } + + AddChangeSender(indexPathId, fullTableId, ESenderType::AsyncIndex); + } + + for (const auto& [streamPathId, _] : tableInfo->CdcStreams) { + AddChangeSender(streamPathId, fullTableId, ESenderType::CdcStream); + } + } + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); + hFunc(TEvChangeExchange::TEvAddSender, Handle); + hFunc(TEvChangeExchange::TEvRemoveSender, Handle); + + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + + STATEFN(StateInactive) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvActivateSender, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + STATEFN(StateActive) { + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + +private: + const TDataShardId DataShard; + mutable TMaybe<TString> LogPrefix; + + THashMap<TPathId, TSender> Senders; + TVector<TEnqueuedRecord> Enqueued; // Enqueued while inactive + +}; // TChangeSender + IActor* CreateChangeSender(const TDataShard* self) { - return new TChangeSender(self); -} - + return new TChangeSender(self); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_sender_async_index.cpp b/ydb/core/tx/datashard/change_sender_async_index.cpp index 1bfab705a14..1bc6796dbf6 100644 --- a/ydb/core/tx/datashard/change_sender_async_index.cpp +++ b/ydb/core/tx/datashard/change_sender_async_index.cpp @@ -1,727 +1,727 @@ -#include "change_exchange.h" -#include "change_exchange_impl.h" -#include "change_sender_common_ops.h" - +#include "change_exchange.h" +#include "change_exchange_impl.h" +#include "change_sender_common_ops.h" + #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/tablet_flat/flat_row_eggs.h> #include <ydb/core/tx/tx_proxy/proxy.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + #include <ydb/library/yql/public/udf/udf_data_type.h> - -#include <util/generic/maybe.h> - -namespace NKikimr { + +#include <util/generic/maybe.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NTable; - -class TAsyncIndexChangeSenderShard: public TActorBootstrapped<TAsyncIndexChangeSenderShard> { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[AsyncIndexChangeSenderShard]" - << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" - << "[" << ShardId << "]" - << SelfId() /* contains brackets */ << " "; - } - - return LogPrefix.GetRef(); - } - - /// GetProxyServices - - void GetProxyServices() { - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); - Become(&TThis::StateGetProxyServices); - } - - STATEFN(StateGetProxyServices) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - + +using namespace NTable; + +class TAsyncIndexChangeSenderShard: public TActorBootstrapped<TAsyncIndexChangeSenderShard> { + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[AsyncIndexChangeSenderShard]" + << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" + << "[" << ShardId << "]" + << SelfId() /* contains brackets */ << " "; + } + + return LogPrefix.GetRef(); + } + + /// GetProxyServices + + void GetProxyServices() { + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest); + Become(&TThis::StateGetProxyServices); + } + + STATEFN(StateGetProxyServices) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + LeaderPipeCache = ev->Get()->Services.LeaderPipeCache; - Handshake(); - } - - /// Handshake - - void Handshake() { - auto ev = MakeHolder<TEvChangeExchange::TEvHandshake>(); - ev->Record.SetOrigin(DataShard.TabletId); - ev->Record.SetGeneration(DataShard.Generation); - + Handshake(); + } + + /// Handshake + + void Handshake() { + auto ev = MakeHolder<TEvChangeExchange::TEvHandshake>(); + ev->Record.SetOrigin(DataShard.TabletId); + ev->Record.SetGeneration(DataShard.Generation); + Send(LeaderPipeCache, new TEvPipeCache::TEvForward(ev.Release(), ShardId, true)); - Become(&TThis::StateHandshake); - } - - STATEFN(StateHandshake) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvStatus, Handshake); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handshake(TEvChangeExchange::TEvStatus::TPtr& ev) { - LOG_D("Handshake " << ev->Get()->ToString()); - - const auto& record = ev->Get()->Record; - switch (record.GetStatus()) { - case NKikimrChangeExchange::TEvStatus::STATUS_OK: - LastRecordOrder = record.GetLastRecordOrder(); - return Ready(); - default: - LOG_E("Handshake status" - << ": status# " << static_cast<ui32>(record.GetStatus()) - << ", reason# " << static_cast<ui32>(record.GetReason())); - return Leave(); - } - } - - void Ready() { - Send(Parent, new TEvChangeExchangePrivate::TEvReady(ShardId)); - Become(&TThis::StateWaitingRecords); - } - - /// WaitingRecords - - STATEFN(StateWaitingRecords) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvRecords, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - auto records = MakeHolder<TEvChangeExchange::TEvApplyRecords>(); - records->Record.SetOrigin(DataShard.TabletId); - records->Record.SetGeneration(DataShard.Generation); - - for (const auto& record : ev->Get()->Records) { - if (record.GetOrder() <= LastRecordOrder) { - continue; - } - - auto& proto = *records->Record.AddRecords(); - record.SerializeTo(proto); - Adjust(proto); - } - - if (!records->Record.RecordsSize()) { - return Ready(); - } - + Become(&TThis::StateHandshake); + } + + STATEFN(StateHandshake) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvStatus, Handshake); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handshake(TEvChangeExchange::TEvStatus::TPtr& ev) { + LOG_D("Handshake " << ev->Get()->ToString()); + + const auto& record = ev->Get()->Record; + switch (record.GetStatus()) { + case NKikimrChangeExchange::TEvStatus::STATUS_OK: + LastRecordOrder = record.GetLastRecordOrder(); + return Ready(); + default: + LOG_E("Handshake status" + << ": status# " << static_cast<ui32>(record.GetStatus()) + << ", reason# " << static_cast<ui32>(record.GetReason())); + return Leave(); + } + } + + void Ready() { + Send(Parent, new TEvChangeExchangePrivate::TEvReady(ShardId)); + Become(&TThis::StateWaitingRecords); + } + + /// WaitingRecords + + STATEFN(StateWaitingRecords) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvRecords, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + auto records = MakeHolder<TEvChangeExchange::TEvApplyRecords>(); + records->Record.SetOrigin(DataShard.TabletId); + records->Record.SetGeneration(DataShard.Generation); + + for (const auto& record : ev->Get()->Records) { + if (record.GetOrder() <= LastRecordOrder) { + continue; + } + + auto& proto = *records->Record.AddRecords(); + record.SerializeTo(proto); + Adjust(proto); + } + + if (!records->Record.RecordsSize()) { + return Ready(); + } + Send(LeaderPipeCache, new TEvPipeCache::TEvForward(records.Release(), ShardId, false)); - Become(&TThis::StateWaitingStatus); - } - - void Adjust(NKikimrChangeExchange::TChangeRecord& record) const { - record.SetPathOwnerId(IndexTablePathId.OwnerId); - record.SetLocalPathId(IndexTablePathId.LocalPathId); - - Y_VERIFY(record.HasAsyncIndex()); - AdjustTags(*record.MutableAsyncIndex()); - } - - void AdjustTags(NKikimrChangeExchange::TChangeRecord::TDataChange& record) const { - AdjustTags(*record.MutableKey()->MutableTags()); - - switch (record.GetRowOperationCase()) { - case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: - AdjustTags(*record.MutableUpsert()->MutableTags()); - break; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: - AdjustTags(*record.MutableReset()->MutableTags()); - break; - default: - break; - } - } - - void AdjustTags(google::protobuf::RepeatedField<ui32>& tags) const { - for (int i = 0; i < tags.size(); ++i) { - auto it = TagMap.find(tags[i]); - Y_VERIFY(it != TagMap.end()); - tags[i] = it->second; - } - } - - /// WaitingStatus - - STATEFN(StateWaitingStatus) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvStatus, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvChangeExchange::TEvStatus::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - const auto& record = ev->Get()->Record; - switch (record.GetStatus()) { - case NKikimrChangeExchange::TEvStatus::STATUS_OK: - LastRecordOrder = record.GetLastRecordOrder(); - return Ready(); - // TODO: REJECT? - default: - LOG_E("Apply status" - << ": status# " << static_cast<ui32>(record.GetStatus()) - << ", reason# " << static_cast<ui32>(record.GetReason())); - return Leave(); - } - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - if (ShardId != ev->Get()->TabletId) { - return; - } - - Leave(); - } - - void Leave() { - Send(Parent, new TEvChangeExchangePrivate::TEvGone(ShardId)); - PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION; - } - - TAsyncIndexChangeSenderShard(const TActorId& parent, const TDataShardId& dataShard, ui64 shardId, - const TPathId& indexTablePathId, const TMap<TTag, TTag>& tagMap) - : Parent(parent) - , DataShard(dataShard) - , ShardId(shardId) - , IndexTablePathId(indexTablePathId) - , TagMap(tagMap) - , LastRecordOrder(0) - { - } - - void Bootstrap() { - GetProxyServices(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const TDataShardId DataShard; - const ui64 ShardId; - const TPathId IndexTablePathId; - const TMap<TTag, TTag> TagMap; // from main to index - mutable TMaybe<TString> LogPrefix; - + Become(&TThis::StateWaitingStatus); + } + + void Adjust(NKikimrChangeExchange::TChangeRecord& record) const { + record.SetPathOwnerId(IndexTablePathId.OwnerId); + record.SetLocalPathId(IndexTablePathId.LocalPathId); + + Y_VERIFY(record.HasAsyncIndex()); + AdjustTags(*record.MutableAsyncIndex()); + } + + void AdjustTags(NKikimrChangeExchange::TChangeRecord::TDataChange& record) const { + AdjustTags(*record.MutableKey()->MutableTags()); + + switch (record.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + AdjustTags(*record.MutableUpsert()->MutableTags()); + break; + case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: + AdjustTags(*record.MutableReset()->MutableTags()); + break; + default: + break; + } + } + + void AdjustTags(google::protobuf::RepeatedField<ui32>& tags) const { + for (int i = 0; i < tags.size(); ++i) { + auto it = TagMap.find(tags[i]); + Y_VERIFY(it != TagMap.end()); + tags[i] = it->second; + } + } + + /// WaitingStatus + + STATEFN(StateWaitingStatus) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvStatus, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvChangeExchange::TEvStatus::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + const auto& record = ev->Get()->Record; + switch (record.GetStatus()) { + case NKikimrChangeExchange::TEvStatus::STATUS_OK: + LastRecordOrder = record.GetLastRecordOrder(); + return Ready(); + // TODO: REJECT? + default: + LOG_E("Apply status" + << ": status# " << static_cast<ui32>(record.GetStatus()) + << ", reason# " << static_cast<ui32>(record.GetReason())); + return Leave(); + } + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + if (ShardId != ev->Get()->TabletId) { + return; + } + + Leave(); + } + + void Leave() { + Send(Parent, new TEvChangeExchangePrivate::TEvGone(ShardId)); + PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION; + } + + TAsyncIndexChangeSenderShard(const TActorId& parent, const TDataShardId& dataShard, ui64 shardId, + const TPathId& indexTablePathId, const TMap<TTag, TTag>& tagMap) + : Parent(parent) + , DataShard(dataShard) + , ShardId(shardId) + , IndexTablePathId(indexTablePathId) + , TagMap(tagMap) + , LastRecordOrder(0) + { + } + + void Bootstrap() { + GetProxyServices(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const TDataShardId DataShard; + const ui64 ShardId; + const TPathId IndexTablePathId; + const TMap<TTag, TTag> TagMap; // from main to index + mutable TMaybe<TString> LogPrefix; + TActorId LeaderPipeCache; - ui64 LastRecordOrder; - -}; // TAsyncIndexChangeSenderShard - -class TAsyncIndexChangeSenderMain: public TActorBootstrapped<TAsyncIndexChangeSenderMain> - , public TBaseChangeSender - , public IChangeSenderResolver - , private TSchemeCacheHelpers { - - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[AsyncIndexChangeSenderMain]" - << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" - << SelfId() /* contains brackets */ << " "; - } - - return LogPrefix.GetRef(); - } - - static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { - TVector<TCell> fromValues(keyColumnsCount); - TVector<TCell> toValues; - return TSerializedTableRange(fromValues, true, toValues, false); - } - - bool IsResolvingUserTable() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveUserTable); - } - - bool IsResolvingIndex() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveIndex); - } - - bool IsResolvingIndexTable() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveIndexTable); - } - - bool IsResolvingKeys() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveKeys); - } - - bool IsResolving() const override { - return IsResolvingUserTable() - || IsResolvingIndex() - || IsResolvingIndexTable() - || IsResolvingKeys(); - } - - TStringBuf CurrentStateName() const { - if (IsResolvingUserTable()) { - return "ResolveUserTable"; - } else if (IsResolvingIndex()) { - return "ResolveIndex"; - } else if (IsResolvingIndexTable()) { - return "ResolveIndexTable"; - } else if (IsResolvingKeys()) { - return "ResolveKeys"; - } else { - return ""; - } - } - - void Retry() { - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); - } - - void LogCritAndRetry(const TString& error) { - LOG_C(error); - Retry(); - } - - void LogWarnAndRetry(const TString& error) { - LOG_W(error); - Retry(); - } - - template <typename CheckFunc, typename FailFunc, typename T, typename... Args> - bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { - return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); - } - - template <typename T> - bool CheckNotEmpty(const TAutoPtr<T>& result) { - return Check(&TSchemeCacheHelpers::CheckNotEmpty<T>, &TThis::LogCritAndRetry, result); - } - - template <typename T> - bool CheckEntriesCount(const TAutoPtr<T>& result, ui32 expected) { - return Check(&TSchemeCacheHelpers::CheckEntriesCount<T>, &TThis::LogCritAndRetry, result, expected); - } - - template <typename T> - bool CheckTableId(const T& entry, const TTableId& expected) { - return Check(&TSchemeCacheHelpers::CheckTableId<T>, &TThis::LogCritAndRetry, entry, expected); - } - - template <typename T> - bool CheckEntrySucceeded(const T& entry) { - return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<T>, &TThis::LogWarnAndRetry, entry); - } - - template <typename T> - bool CheckEntryKind(const T& entry, TNavigate::EKind expected) { - return Check(&TSchemeCacheHelpers::CheckEntryKind<T>, &TThis::LogWarnAndRetry, entry, expected); - } - - static TVector<ui64> MakePartitionIds(const TVector<TKeyDesc::TPartitionInfo>& partitions) { - TVector<ui64> result(Reserve(partitions.size())); - - for (const auto& partition : partitions) { - result.push_back(partition.ShardId); // partition = shard - } - - return result; - } - - /// ResolveUserTable - - void ResolveUserTable() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(UserTableId, TNavigate::OpTable)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveUserTable); - } - - STATEFN(StateResolveUserTable) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleUserTable); - sFunc(TEvents::TEvWakeup, ResolveUserTable); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleUserTable(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - + ui64 LastRecordOrder; + +}; // TAsyncIndexChangeSenderShard + +class TAsyncIndexChangeSenderMain: public TActorBootstrapped<TAsyncIndexChangeSenderMain> + , public TBaseChangeSender + , public IChangeSenderResolver + , private TSchemeCacheHelpers { + + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[AsyncIndexChangeSenderMain]" + << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" + << SelfId() /* contains brackets */ << " "; + } + + return LogPrefix.GetRef(); + } + + static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { + TVector<TCell> fromValues(keyColumnsCount); + TVector<TCell> toValues; + return TSerializedTableRange(fromValues, true, toValues, false); + } + + bool IsResolvingUserTable() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveUserTable); + } + + bool IsResolvingIndex() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveIndex); + } + + bool IsResolvingIndexTable() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveIndexTable); + } + + bool IsResolvingKeys() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveKeys); + } + + bool IsResolving() const override { + return IsResolvingUserTable() + || IsResolvingIndex() + || IsResolvingIndexTable() + || IsResolvingKeys(); + } + + TStringBuf CurrentStateName() const { + if (IsResolvingUserTable()) { + return "ResolveUserTable"; + } else if (IsResolvingIndex()) { + return "ResolveIndex"; + } else if (IsResolvingIndexTable()) { + return "ResolveIndexTable"; + } else if (IsResolvingKeys()) { + return "ResolveKeys"; + } else { + return ""; + } + } + + void Retry() { + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); + } + + void LogCritAndRetry(const TString& error) { + LOG_C(error); + Retry(); + } + + void LogWarnAndRetry(const TString& error) { + LOG_W(error); + Retry(); + } + + template <typename CheckFunc, typename FailFunc, typename T, typename... Args> + bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { + return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); + } + + template <typename T> + bool CheckNotEmpty(const TAutoPtr<T>& result) { + return Check(&TSchemeCacheHelpers::CheckNotEmpty<T>, &TThis::LogCritAndRetry, result); + } + + template <typename T> + bool CheckEntriesCount(const TAutoPtr<T>& result, ui32 expected) { + return Check(&TSchemeCacheHelpers::CheckEntriesCount<T>, &TThis::LogCritAndRetry, result, expected); + } + + template <typename T> + bool CheckTableId(const T& entry, const TTableId& expected) { + return Check(&TSchemeCacheHelpers::CheckTableId<T>, &TThis::LogCritAndRetry, entry, expected); + } + + template <typename T> + bool CheckEntrySucceeded(const T& entry) { + return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<T>, &TThis::LogWarnAndRetry, entry); + } + + template <typename T> + bool CheckEntryKind(const T& entry, TNavigate::EKind expected) { + return Check(&TSchemeCacheHelpers::CheckEntryKind<T>, &TThis::LogWarnAndRetry, entry, expected); + } + + static TVector<ui64> MakePartitionIds(const TVector<TKeyDesc::TPartitionInfo>& partitions) { + TVector<ui64> result(Reserve(partitions.size())); + + for (const auto& partition : partitions) { + result.push_back(partition.ShardId); // partition = shard + } + + return result; + } + + /// ResolveUserTable + + void ResolveUserTable() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(UserTableId, TNavigate::OpTable)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveUserTable); + } + + STATEFN(StateResolveUserTable) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleUserTable); + sFunc(TEvents::TEvWakeup, ResolveUserTable); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleUserTable(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + LOG_D("HandleUserTable TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, UserTableId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindTable)) { - return; - } - - for (const auto& [tag, column] : entry.Columns) { - Y_VERIFY_DEBUG(!MainColumnToTag.contains(column.Name)); - MainColumnToTag.emplace(column.Name, tag); - } - - ResolveIndex(); - } - - /// ResolveIndex - - void ResolveIndex() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveIndex); - } - - STATEFN(StateResolveIndex) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleIndex); - sFunc(TEvents::TEvWakeup, ResolveIndex); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleIndex(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("HandleIndex TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, PathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindIndex)) { - return; - } - - Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); - const auto& indexTable = entry.ListNodeEntry->Children.at(0); - - Y_VERIFY(indexTable.Kind == TNavigate::KindTable); - IndexTablePathId = indexTable.PathId; - - ResolveIndexTable(); - } - - /// ResolveIndexTable - - void ResolveIndexTable() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(IndexTablePathId, TNavigate::OpTable)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveIndexTable); - } - - STATEFN(StateResolveIndexTable) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleIndexTable); - sFunc(TEvents::TEvWakeup, ResolveIndexTable); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleIndexTable(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("HandleIndexTable TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, IndexTablePathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindTable)) { - return; - } - - TagMap.clear(); - TVector<NYql::NUdf::TDataTypeId> keyColumnTypes; - - for (const auto& [tag, column] : entry.Columns) { - auto it = MainColumnToTag.find(column.Name); - Y_VERIFY(it != MainColumnToTag.end()); - - Y_VERIFY_DEBUG(!TagMap.contains(it->second)); - TagMap.emplace(it->second, tag); - - if (column.KeyOrder < 0) { - continue; - } - - if (keyColumnTypes.size() <= static_cast<ui32>(column.KeyOrder)) { - keyColumnTypes.resize(column.KeyOrder + 1); - } - - keyColumnTypes[column.KeyOrder] = column.PType; - } - - KeyDesc = MakeHolder<TKeyDesc>( - entry.TableId, - GetFullRange(keyColumnTypes.size()).ToTableRange(), - TKeyDesc::ERowOperation::Update, - keyColumnTypes, - TVector<TKeyDesc::TColumnOp>() - ); - - ResolveKeys(); - } - - /// ResolveKeys - - void ResolveKeys() { - auto request = MakeHolder<TResolve>(); - request->ResultSet.emplace_back(std::move(KeyDesc)); - - Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); - Become(&TThis::StateResolveKeys); - } - - STATEFN(StateResolveKeys) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleKeys); - sFunc(TEvents::TEvWakeup, ResolveIndexTable); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleKeys(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("HandleKeys TEvTxProxySchemeCache::TEvResolveKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, IndexTablePathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!entry.KeyDescription->Partitions) { - LOG_W("Empty partitions list" - << ": entry# " << entry.ToString(*AppData()->TypeRegistry)); - return Retry(); - } - - KeyDesc = std::move(entry.KeyDescription); - CreateSenders(MakePartitionIds(KeyDesc->Partitions)); - - Become(&TThis::StateMain); - } - - /// Main - - STATEFN(StateMain) { - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - - void Resolve() override { - ResolveIndexTable(); - } - - bool IsResolved() const override { - return KeyDesc && KeyDesc->Partitions; - } - - ui64 GetPartitionId(TConstArrayRef<TCell> key) const override { - Y_VERIFY(KeyDesc); - Y_VERIFY(KeyDesc->Partitions); - - const auto range = TTableRange(key); - Y_VERIFY(range.Point); - - TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( - KeyDesc->Partitions.begin(), KeyDesc->Partitions.end(), true, - [&](const TKeyDesc::TPartitionInfo& partition, bool) { - const int compares = CompareBorders<true, false>( - partition.Range->EndKeyPrefix.GetCells(), range.From, - partition.Range->IsInclusive || partition.Range->IsPoint, - range.InclusiveFrom || range.Point, KeyDesc->KeyColumnTypes - ); - - return (compares < 0); - } - ); - - Y_VERIFY(it != KeyDesc->Partitions.end()); - return it->ShardId; // partition = shard - } - - IActor* CreateSender(ui64 partitionId) override { - return new TAsyncIndexChangeSenderShard(SelfId(), DataShard, partitionId, IndexTablePathId, TagMap); - } - - void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - EnqueueRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - ProcessRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchange::TEvForgetRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - ForgetRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchangePrivate::TEvReady::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - OnReady(ev->Get()->PartitionId); - } - - void Handle(TEvChangeExchangePrivate::TEvGone::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - OnGone(ev->Get()->PartitionId); - } - - void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - Y_VERIFY(ev->Get()->PathId == PathId); - - RemoveRecords(); - PassAway(); - } - - void PassAway() override { - KillSenders(); - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN; - } - - explicit TAsyncIndexChangeSenderMain(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId) - : TActorBootstrapped() - , TBaseChangeSender(this, this, dataShard, indexPathId) - , UserTableId(userTableId) - { - } - - void Bootstrap() { - ResolveUserTable(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); - hFunc(TEvChangeExchange::TEvRecords, Handle); - hFunc(TEvChangeExchange::TEvForgetRecords, Handle); - hFunc(TEvChangeExchange::TEvRemoveSender, Handle); - hFunc(TEvChangeExchangePrivate::TEvReady, Handle); - hFunc(TEvChangeExchangePrivate::TEvGone, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TTableId UserTableId; - mutable TMaybe<TString> LogPrefix; - - THashMap<TString, TTag> MainColumnToTag; - TMap<TTag, TTag> TagMap; // from main to index - - TPathId IndexTablePathId; - THolder<TKeyDesc> KeyDesc; - -}; // TAsyncIndexChangeSenderMain - -IActor* CreateAsyncIndexChangeSender(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId) { - return new TAsyncIndexChangeSenderMain(dataShard, userTableId, indexPathId); -} - + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, UserTableId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindTable)) { + return; + } + + for (const auto& [tag, column] : entry.Columns) { + Y_VERIFY_DEBUG(!MainColumnToTag.contains(column.Name)); + MainColumnToTag.emplace(column.Name, tag); + } + + ResolveIndex(); + } + + /// ResolveIndex + + void ResolveIndex() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveIndex); + } + + STATEFN(StateResolveIndex) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleIndex); + sFunc(TEvents::TEvWakeup, ResolveIndex); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleIndex(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("HandleIndex TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, PathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindIndex)) { + return; + } + + Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); + const auto& indexTable = entry.ListNodeEntry->Children.at(0); + + Y_VERIFY(indexTable.Kind == TNavigate::KindTable); + IndexTablePathId = indexTable.PathId; + + ResolveIndexTable(); + } + + /// ResolveIndexTable + + void ResolveIndexTable() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(IndexTablePathId, TNavigate::OpTable)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveIndexTable); + } + + STATEFN(StateResolveIndexTable) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleIndexTable); + sFunc(TEvents::TEvWakeup, ResolveIndexTable); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleIndexTable(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("HandleIndexTable TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, IndexTablePathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindTable)) { + return; + } + + TagMap.clear(); + TVector<NYql::NUdf::TDataTypeId> keyColumnTypes; + + for (const auto& [tag, column] : entry.Columns) { + auto it = MainColumnToTag.find(column.Name); + Y_VERIFY(it != MainColumnToTag.end()); + + Y_VERIFY_DEBUG(!TagMap.contains(it->second)); + TagMap.emplace(it->second, tag); + + if (column.KeyOrder < 0) { + continue; + } + + if (keyColumnTypes.size() <= static_cast<ui32>(column.KeyOrder)) { + keyColumnTypes.resize(column.KeyOrder + 1); + } + + keyColumnTypes[column.KeyOrder] = column.PType; + } + + KeyDesc = MakeHolder<TKeyDesc>( + entry.TableId, + GetFullRange(keyColumnTypes.size()).ToTableRange(), + TKeyDesc::ERowOperation::Update, + keyColumnTypes, + TVector<TKeyDesc::TColumnOp>() + ); + + ResolveKeys(); + } + + /// ResolveKeys + + void ResolveKeys() { + auto request = MakeHolder<TResolve>(); + request->ResultSet.emplace_back(std::move(KeyDesc)); + + Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); + Become(&TThis::StateResolveKeys); + } + + STATEFN(StateResolveKeys) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleKeys); + sFunc(TEvents::TEvWakeup, ResolveIndexTable); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleKeys(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("HandleKeys TEvTxProxySchemeCache::TEvResolveKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, IndexTablePathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!entry.KeyDescription->Partitions) { + LOG_W("Empty partitions list" + << ": entry# " << entry.ToString(*AppData()->TypeRegistry)); + return Retry(); + } + + KeyDesc = std::move(entry.KeyDescription); + CreateSenders(MakePartitionIds(KeyDesc->Partitions)); + + Become(&TThis::StateMain); + } + + /// Main + + STATEFN(StateMain) { + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + + void Resolve() override { + ResolveIndexTable(); + } + + bool IsResolved() const override { + return KeyDesc && KeyDesc->Partitions; + } + + ui64 GetPartitionId(TConstArrayRef<TCell> key) const override { + Y_VERIFY(KeyDesc); + Y_VERIFY(KeyDesc->Partitions); + + const auto range = TTableRange(key); + Y_VERIFY(range.Point); + + TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( + KeyDesc->Partitions.begin(), KeyDesc->Partitions.end(), true, + [&](const TKeyDesc::TPartitionInfo& partition, bool) { + const int compares = CompareBorders<true, false>( + partition.Range->EndKeyPrefix.GetCells(), range.From, + partition.Range->IsInclusive || partition.Range->IsPoint, + range.InclusiveFrom || range.Point, KeyDesc->KeyColumnTypes + ); + + return (compares < 0); + } + ); + + Y_VERIFY(it != KeyDesc->Partitions.end()); + return it->ShardId; // partition = shard + } + + IActor* CreateSender(ui64 partitionId) override { + return new TAsyncIndexChangeSenderShard(SelfId(), DataShard, partitionId, IndexTablePathId, TagMap); + } + + void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + EnqueueRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + ProcessRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchange::TEvForgetRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + ForgetRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchangePrivate::TEvReady::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + OnReady(ev->Get()->PartitionId); + } + + void Handle(TEvChangeExchangePrivate::TEvGone::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + OnGone(ev->Get()->PartitionId); + } + + void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + Y_VERIFY(ev->Get()->PathId == PathId); + + RemoveRecords(); + PassAway(); + } + + void PassAway() override { + KillSenders(); + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN; + } + + explicit TAsyncIndexChangeSenderMain(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId) + : TActorBootstrapped() + , TBaseChangeSender(this, this, dataShard, indexPathId) + , UserTableId(userTableId) + { + } + + void Bootstrap() { + ResolveUserTable(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); + hFunc(TEvChangeExchange::TEvRecords, Handle); + hFunc(TEvChangeExchange::TEvForgetRecords, Handle); + hFunc(TEvChangeExchange::TEvRemoveSender, Handle); + hFunc(TEvChangeExchangePrivate::TEvReady, Handle); + hFunc(TEvChangeExchangePrivate::TEvGone, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TTableId UserTableId; + mutable TMaybe<TString> LogPrefix; + + THashMap<TString, TTag> MainColumnToTag; + TMap<TTag, TTag> TagMap; // from main to index + + TPathId IndexTablePathId; + THolder<TKeyDesc> KeyDesc; + +}; // TAsyncIndexChangeSenderMain + +IActor* CreateAsyncIndexChangeSender(const TDataShardId& dataShard, const TTableId& userTableId, const TPathId& indexPathId) { + return new TAsyncIndexChangeSenderMain(dataShard, userTableId, indexPathId); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp index 9ca17f862df..288f6a8526f 100644 --- a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp +++ b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp @@ -1,682 +1,682 @@ -#include "change_exchange.h" -#include "change_exchange_impl.h" -#include "change_sender_common_ops.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - +#include "change_exchange.h" +#include "change_exchange_impl.h" +#include "change_sender_common_ops.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + #include <ydb/core/persqueue/partition_key_range/partition_key_range.h> #include <ydb/core/persqueue/writer/source_id_encoding.h> #include <ydb/core/persqueue/writer/writer.h> #include <ydb/core/protos/grpc_pq_old.pb.h> - -namespace NKikimr { -namespace NDataShard { - -using namespace NPQ; - -class TCdcChangeSenderPartition: public TActorBootstrapped<TCdcChangeSenderPartition> { - static constexpr auto CodecRaw = 1; - - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[CdcChangeSenderPartition]" - << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" - << "[" << PartitionId << "]" - << "[" << ShardId << "]" - << SelfId() /* contains brackets */ << " "; - } - - return LogPrefix.GetRef(); - } - - /// Init - - STATEFN(StateInit) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPartitionWriter::TEvInitResult, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvPartitionWriter::TEvInitResult::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - const auto& result = *ev->Get(); - if (!result.IsSuccess()) { - LOG_E("Error at 'Init'" - << ": reason# " << result.GetError().Reason); - return Leave(); - } - - const auto& info = result.GetResult().SourceIdInfo; - Y_VERIFY(info.GetExplicit()); - - MaxSeqNo = info.GetSeqNo(); - Ready(); - } - - void Ready() { - Pending.clear(); - - Send(Parent, new TEvChangeExchangePrivate::TEvReady(PartitionId)); - Become(&TThis::StateWaitingRecords); - } - - /// WaitingRecords - - STATEFN(StateWaitingRecords) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvRecords, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - NKikimrClient::TPersQueueRequest request; - - for (const auto& record : ev->Get()->Records) { - if (record.GetSeqNo() <= MaxSeqNo) { - continue; - } - - const auto createdAt = TInstant::FromValue(record.GetGroup()); - - NKikimrChangeExchange::TChangeRecord protoRecord; - record.SerializeTo(protoRecord); - - NKikimrPQClient::TDataChunk data; - data.SetSeqNo(record.GetSeqNo()); - data.SetCreateTime(createdAt.MilliSeconds()); - data.SetCodec(CodecRaw); - data.SetData(protoRecord.SerializeAsString()); - // TODO: meta? - - auto& cmd = *request.MutablePartitionRequest()->AddCmdWrite(); - cmd.SetSeqNo(record.GetSeqNo()); - cmd.SetSourceId(NSourceIdEncoding::EncodeSimple(SourceId)); - cmd.SetCreateTimeMS(createdAt.MilliSeconds()); - cmd.SetData(data.SerializeAsString()); - - Pending.push_back(record.GetSeqNo()); - } - - if (!Pending) { - return Ready(); - } - - Write(std::move(request)); - } - - /// Write - - void Write(NKikimrClient::TPersQueueRequest&& request) { - auto ev = MakeHolder<TEvPartitionWriter::TEvWriteRequest>(++Cookie); - ev->Record = std::move(request); - - Send(Writer, std::move(ev)); - Become(&TThis::StateWrite); - } - - STATEFN(StateWrite) { - switch (ev->GetTypeRewrite()) { - IgnoreFunc(TEvPartitionWriter::TEvWriteAccepted); - hFunc(TEvPartitionWriter::TEvWriteResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvPartitionWriter::TEvWriteResponse::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - - const auto& result = *ev->Get(); - if (!result.IsSuccess()) { - LOG_E("Error at 'Write" - << ": reason# " << result.GetError().Reason); - return Leave(); - } - - const auto& response = result.Record.GetPartitionResponse(); - if (response.GetCookie() != Cookie) { - LOG_E("Cookie mismatch" - << ": expected# " << Cookie - << ", got# " << response.GetCookie()); - return Leave(); - } - - if (response.CmdWriteResultSize() != Pending.size()) { - LOG_E("Write result size mismatch" - << ": expected# " << Pending.size() - << ", got# " << response.CmdWriteResultSize()); - return Leave(); - } - - for (ui32 i = 0; i < Pending.size(); ++i) { - const auto expected = Pending.at(i); - const auto got = response.GetCmdWriteResult(i).GetSeqNo(); - - if (expected == got) { - MaxSeqNo = Max(MaxSeqNo, got); - continue; - } - - LOG_E("SeqNo mismatch" - << ": expected# " << expected - << ", got# " << got); - return Leave(); - } - - Ready(); - } - - void Leave() { - Send(Parent, new TEvChangeExchangePrivate::TEvGone(PartitionId)); - PassAway(); - } - - void PassAway() override { - if (Writer) { - Send(Writer, new TEvents::TEvPoisonPill()); - } - - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_SENDER_CDC_ACTOR_PARTITION; - } - - explicit TCdcChangeSenderPartition(const TActorId& parent, const TDataShardId& dataShard, ui32 partitionId, ui64 shardId) - : Parent(parent) - , DataShard(dataShard) - , PartitionId(partitionId) - , ShardId(shardId) - , SourceId(ToString(DataShard.TabletId)) - { - } - - void Bootstrap() { - auto opts = TPartitionWriterOpts() - .WithCheckState(true) - .WithAutoRegister(true); - Writer = RegisterWithSameMailbox(CreatePartitionWriter(SelfId(), ShardId, PartitionId, SourceId, opts)); - Become(&TThis::StateInit); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvPartitionWriter::TEvDisconnected, Leave); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const TDataShardId DataShard; - const ui32 PartitionId; - const ui64 ShardId; - const TString SourceId; - mutable TMaybe<TString> LogPrefix; - - TActorId Writer; - i64 MaxSeqNo = 0; - TVector<i64> Pending; - ui64 Cookie = 0; - -}; // TCdcChangeSenderPartition - -class TCdcChangeSenderMain: public TActorBootstrapped<TCdcChangeSenderMain> - , public TBaseChangeSender - , public IChangeSenderResolver - , private TSchemeCacheHelpers { - - struct TPQPartitionInfo { - ui32 PartitionId; - ui64 ShardId; - TPartitionKeyRange KeyRange; - - struct TLess { - TConstArrayRef<NScheme::TTypeId> Schema; - - TLess(const TVector<NScheme::TTypeId>& schema) - : Schema(schema) - { - } - - bool operator()(const TPQPartitionInfo& lhs, const TPQPartitionInfo& rhs) const { - Y_VERIFY(lhs.KeyRange.ToBound || rhs.KeyRange.ToBound); - - if (!lhs.KeyRange.ToBound) { - return false; - } - - if (!rhs.KeyRange.ToBound) { - return true; - } - - Y_VERIFY(lhs.KeyRange.ToBound && rhs.KeyRange.ToBound); - - const int compares = CompareTypedCellVectors( - lhs.KeyRange.ToBound->GetCells().data(), - rhs.KeyRange.ToBound->GetCells().data(), - Schema.data(), Schema.size() - ); - - return (compares < 0); - } - - }; // TLess - - }; // TPQPartitionInfo - - struct TKeyDesc { - struct TPartitionInfo { - ui32 PartitionId; - ui64 ShardId; - TSerializedCellVec EndKeyPrefix; - // just a hint - static constexpr bool IsInclusive = false; - static constexpr bool IsPoint = false; - - explicit TPartitionInfo(const TPQPartitionInfo& info) - : PartitionId(info.PartitionId) - , ShardId(info.ShardId) - { - if (info.KeyRange.ToBound) { - EndKeyPrefix = *info.KeyRange.ToBound; - } - } - - }; // TPartitionInfo - - TVector<NScheme::TTypeId> Schema; - TVector<TPartitionInfo> Partitions; - - }; // TKeyDesc - - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[CdcChangeSenderMain]" - << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" - << SelfId() /* contains brackets */ << " "; - } - - return LogPrefix.GetRef(); - } - - bool IsResolvingCdcStream() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveCdcStream); - } - - bool IsResolvingTopic() const { - return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveTopic); - } - - bool IsResolving() const override { - return IsResolvingCdcStream() - || IsResolvingTopic(); - } - - TStringBuf CurrentStateName() const { - if (IsResolvingCdcStream()) { - return "ResolveCdcStream"; - } else if (IsResolvingTopic()) { - return "ResolveTopic"; - } else { - return ""; - } - } - - void Retry() { - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); - } - - void LogCritAndRetry(const TString& error) { - LOG_C(error); - Retry(); - } - - void LogWarnAndRetry(const TString& error) { - LOG_W(error); - Retry(); - } - - template <typename CheckFunc, typename FailFunc, typename T, typename... Args> - bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { - return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); - } - - bool CheckNotEmpty(const TAutoPtr<TNavigate>& result) { - return Check(&TSchemeCacheHelpers::CheckNotEmpty<TNavigate>, &TThis::LogCritAndRetry, result); - } - - bool CheckEntriesCount(const TAutoPtr<TNavigate>& result, ui32 expected) { - return Check(&TSchemeCacheHelpers::CheckEntriesCount<TNavigate>, &TThis::LogCritAndRetry, result, expected); - } - - bool CheckTableId(const TNavigate::TEntry& entry, const TTableId& expected) { - return Check(&TSchemeCacheHelpers::CheckTableId<TNavigate::TEntry>, &TThis::LogCritAndRetry, entry, expected); - } - - bool CheckEntrySucceeded(const TNavigate::TEntry& entry) { - return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry); - } - - bool CheckEntryKind(const TNavigate::TEntry& entry, TNavigate::EKind expected) { - return Check(&TSchemeCacheHelpers::CheckEntryKind<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry, expected); - } - - bool CheckNotEmpty(const TIntrusiveConstPtr<TNavigate::TPQGroupInfo>& pqInfo) { - if (pqInfo) { - return true; - } - - LogCritAndRetry(TStringBuilder() << "Empty pq info at '" << CurrentStateName() << "'"); - return false; - } - - static TVector<ui64> MakePartitionIds(const TVector<TKeyDesc::TPartitionInfo>& partitions) { - TVector<ui64> result(Reserve(partitions.size())); - - for (const auto& partition : partitions) { - result.push_back(partition.PartitionId); - } - - return result; - } - - /// ResolveCdcStream - - void ResolveCdcStream() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveCdcStream); - } - - STATEFN(StateResolveCdcStream) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleCdcStream); - sFunc(TEvents::TEvWakeup, ResolveCdcStream); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleCdcStream(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, PathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindCdcStream)) { - return; - } - - Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); - const auto& topic = entry.ListNodeEntry->Children.at(0); - - Y_VERIFY(topic.Kind == TNavigate::KindTopic); - TopicPathId = topic.PathId; - - ResolveTopic(); - } - - /// ResolveTopic - - void ResolveTopic() { - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(TopicPathId, TNavigate::OpTopic)); - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveTopic); - } - - STATEFN(StateResolveTopic) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTopic); - sFunc(TEvents::TEvWakeup, ResolveTopic); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandleTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& result = ev->Get()->Request; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(result)) { - return; - } - - if (!CheckEntriesCount(result, 1)) { - return; - } - - const auto& entry = result->ResultSet.at(0); - - if (!CheckTableId(entry, TopicPathId)) { - return; - } - - if (!CheckEntrySucceeded(entry)) { - return; - } - - if (!CheckEntryKind(entry, TNavigate::KindTopic)) { - return; - } - - if (!CheckNotEmpty(entry.PQGroupInfo)) { - return; - } - - const auto& pqDesc = entry.PQGroupInfo->Description; - const auto& pqConfig = pqDesc.GetPQTabletConfig(); - - KeyDesc = MakeHolder<TKeyDesc>(); - PartitionToShard.clear(); - - KeyDesc->Schema.reserve(pqConfig.PartitionKeySchemaSize()); - for (const auto& keySchema : pqConfig.GetPartitionKeySchema()) { - KeyDesc->Schema.push_back(keySchema.GetTypeId()); - } - - TSet<TPQPartitionInfo, TPQPartitionInfo::TLess> partitions(KeyDesc->Schema); - THashSet<ui64> shards; - - for (const auto& partition : pqDesc.GetPartitions()) { - const auto partitionId = partition.GetPartitionId(); - const auto shardId = partition.GetTabletId(); - - PartitionToShard.emplace(partitionId, shardId); - - auto keyRange = TPartitionKeyRange::Parse(partition.GetKeyRange()); - Y_VERIFY(!keyRange.FromBound || keyRange.FromBound->GetCells().size() == KeyDesc->Schema.size()); - Y_VERIFY(!keyRange.ToBound || keyRange.ToBound->GetCells().size() == KeyDesc->Schema.size()); - - partitions.insert({partitionId, shardId, std::move(keyRange)}); - shards.insert(shardId); - } - - // used to validate - bool isFirst = true; - const TPQPartitionInfo* prev = nullptr; - - KeyDesc->Partitions.reserve(partitions.size()); - for (const auto& cur : partitions) { - if (isFirst) { - isFirst = false; - Y_VERIFY(!cur.KeyRange.FromBound.Defined()); - } else { - Y_VERIFY(cur.KeyRange.FromBound.Defined()); - Y_VERIFY(prev); - Y_VERIFY(prev->KeyRange.ToBound.Defined()); - // TODO: compare cells - } - - KeyDesc->Partitions.emplace_back(cur); - prev = &cur; - } - - if (prev) { - Y_VERIFY(!prev->KeyRange.ToBound.Defined()); - } - - CreateSenders(MakePartitionIds(KeyDesc->Partitions)); - Become(&TThis::StateMain); - } - - /// Main - - STATEFN(StateMain) { - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - - void Resolve() override { - ResolveTopic(); - } - - bool IsResolved() const override { - return KeyDesc && KeyDesc->Partitions; - } - - ui64 GetPartitionId(TConstArrayRef<TCell> key) const override { - Y_VERIFY(KeyDesc); - Y_VERIFY(KeyDesc->Partitions); - - const auto range = TTableRange(key); - Y_VERIFY(range.Point); - - TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( - KeyDesc->Partitions.begin(), KeyDesc->Partitions.end(), true, - [&](const TKeyDesc::TPartitionInfo& partition, bool) { - const int compares = CompareBorders<true, false>( - partition.EndKeyPrefix.GetCells(), range.From, - partition.IsInclusive || partition.IsPoint, - range.InclusiveFrom || range.Point, KeyDesc->Schema - ); - - return (compares < 0); - } - ); - - Y_VERIFY(it != KeyDesc->Partitions.end()); - return it->PartitionId; - } - - IActor* CreateSender(ui64 partitionId) override { - Y_VERIFY(PartitionToShard.contains(partitionId)); - const auto shardId = PartitionToShard.at(partitionId); - return new TCdcChangeSenderPartition(SelfId(), DataShard, partitionId, shardId); - } - - void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - EnqueueRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - ProcessRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchange::TEvForgetRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - ForgetRecords(std::move(ev->Get()->Records)); - } - - void Handle(TEvChangeExchangePrivate::TEvReady::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - OnReady(ev->Get()->PartitionId); - } - - void Handle(TEvChangeExchangePrivate::TEvGone::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - OnGone(ev->Get()->PartitionId); - } - - void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); - Y_VERIFY(ev->Get()->PathId == PathId); - - RemoveRecords(); - PassAway(); - } - - void PassAway() override { - KillSenders(); - TActorBootstrapped::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::CHANGE_SENDER_CDC_ACTOR_MAIN; - } - - explicit TCdcChangeSenderMain(const TDataShardId& dataShard, const TPathId& streamPathId) - : TActorBootstrapped() - , TBaseChangeSender(this, this, dataShard, streamPathId) - { - } - - void Bootstrap() { - ResolveCdcStream(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); - hFunc(TEvChangeExchange::TEvRecords, Handle); - hFunc(TEvChangeExchange::TEvForgetRecords, Handle); - hFunc(TEvChangeExchange::TEvRemoveSender, Handle); - hFunc(TEvChangeExchangePrivate::TEvReady, Handle); - hFunc(TEvChangeExchangePrivate::TEvGone, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - mutable TMaybe<TString> LogPrefix; - - TPathId TopicPathId; - THolder<TKeyDesc> KeyDesc; - THashMap<ui32, ui64> PartitionToShard; - -}; // TCdcChangeSenderMain - -IActor* CreateCdcStreamChangeSender(const TDataShardId& dataShard, const TPathId& streamPathId) { - return new TCdcChangeSenderMain(dataShard, streamPathId); -} - -} // NDataShard -} // NKikimr + +namespace NKikimr { +namespace NDataShard { + +using namespace NPQ; + +class TCdcChangeSenderPartition: public TActorBootstrapped<TCdcChangeSenderPartition> { + static constexpr auto CodecRaw = 1; + + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[CdcChangeSenderPartition]" + << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" + << "[" << PartitionId << "]" + << "[" << ShardId << "]" + << SelfId() /* contains brackets */ << " "; + } + + return LogPrefix.GetRef(); + } + + /// Init + + STATEFN(StateInit) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPartitionWriter::TEvInitResult, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvPartitionWriter::TEvInitResult::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + const auto& result = *ev->Get(); + if (!result.IsSuccess()) { + LOG_E("Error at 'Init'" + << ": reason# " << result.GetError().Reason); + return Leave(); + } + + const auto& info = result.GetResult().SourceIdInfo; + Y_VERIFY(info.GetExplicit()); + + MaxSeqNo = info.GetSeqNo(); + Ready(); + } + + void Ready() { + Pending.clear(); + + Send(Parent, new TEvChangeExchangePrivate::TEvReady(PartitionId)); + Become(&TThis::StateWaitingRecords); + } + + /// WaitingRecords + + STATEFN(StateWaitingRecords) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvRecords, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + NKikimrClient::TPersQueueRequest request; + + for (const auto& record : ev->Get()->Records) { + if (record.GetSeqNo() <= MaxSeqNo) { + continue; + } + + const auto createdAt = TInstant::FromValue(record.GetGroup()); + + NKikimrChangeExchange::TChangeRecord protoRecord; + record.SerializeTo(protoRecord); + + NKikimrPQClient::TDataChunk data; + data.SetSeqNo(record.GetSeqNo()); + data.SetCreateTime(createdAt.MilliSeconds()); + data.SetCodec(CodecRaw); + data.SetData(protoRecord.SerializeAsString()); + // TODO: meta? + + auto& cmd = *request.MutablePartitionRequest()->AddCmdWrite(); + cmd.SetSeqNo(record.GetSeqNo()); + cmd.SetSourceId(NSourceIdEncoding::EncodeSimple(SourceId)); + cmd.SetCreateTimeMS(createdAt.MilliSeconds()); + cmd.SetData(data.SerializeAsString()); + + Pending.push_back(record.GetSeqNo()); + } + + if (!Pending) { + return Ready(); + } + + Write(std::move(request)); + } + + /// Write + + void Write(NKikimrClient::TPersQueueRequest&& request) { + auto ev = MakeHolder<TEvPartitionWriter::TEvWriteRequest>(++Cookie); + ev->Record = std::move(request); + + Send(Writer, std::move(ev)); + Become(&TThis::StateWrite); + } + + STATEFN(StateWrite) { + switch (ev->GetTypeRewrite()) { + IgnoreFunc(TEvPartitionWriter::TEvWriteAccepted); + hFunc(TEvPartitionWriter::TEvWriteResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvPartitionWriter::TEvWriteResponse::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + + const auto& result = *ev->Get(); + if (!result.IsSuccess()) { + LOG_E("Error at 'Write" + << ": reason# " << result.GetError().Reason); + return Leave(); + } + + const auto& response = result.Record.GetPartitionResponse(); + if (response.GetCookie() != Cookie) { + LOG_E("Cookie mismatch" + << ": expected# " << Cookie + << ", got# " << response.GetCookie()); + return Leave(); + } + + if (response.CmdWriteResultSize() != Pending.size()) { + LOG_E("Write result size mismatch" + << ": expected# " << Pending.size() + << ", got# " << response.CmdWriteResultSize()); + return Leave(); + } + + for (ui32 i = 0; i < Pending.size(); ++i) { + const auto expected = Pending.at(i); + const auto got = response.GetCmdWriteResult(i).GetSeqNo(); + + if (expected == got) { + MaxSeqNo = Max(MaxSeqNo, got); + continue; + } + + LOG_E("SeqNo mismatch" + << ": expected# " << expected + << ", got# " << got); + return Leave(); + } + + Ready(); + } + + void Leave() { + Send(Parent, new TEvChangeExchangePrivate::TEvGone(PartitionId)); + PassAway(); + } + + void PassAway() override { + if (Writer) { + Send(Writer, new TEvents::TEvPoisonPill()); + } + + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_SENDER_CDC_ACTOR_PARTITION; + } + + explicit TCdcChangeSenderPartition(const TActorId& parent, const TDataShardId& dataShard, ui32 partitionId, ui64 shardId) + : Parent(parent) + , DataShard(dataShard) + , PartitionId(partitionId) + , ShardId(shardId) + , SourceId(ToString(DataShard.TabletId)) + { + } + + void Bootstrap() { + auto opts = TPartitionWriterOpts() + .WithCheckState(true) + .WithAutoRegister(true); + Writer = RegisterWithSameMailbox(CreatePartitionWriter(SelfId(), ShardId, PartitionId, SourceId, opts)); + Become(&TThis::StateInit); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + sFunc(TEvPartitionWriter::TEvDisconnected, Leave); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const TDataShardId DataShard; + const ui32 PartitionId; + const ui64 ShardId; + const TString SourceId; + mutable TMaybe<TString> LogPrefix; + + TActorId Writer; + i64 MaxSeqNo = 0; + TVector<i64> Pending; + ui64 Cookie = 0; + +}; // TCdcChangeSenderPartition + +class TCdcChangeSenderMain: public TActorBootstrapped<TCdcChangeSenderMain> + , public TBaseChangeSender + , public IChangeSenderResolver + , private TSchemeCacheHelpers { + + struct TPQPartitionInfo { + ui32 PartitionId; + ui64 ShardId; + TPartitionKeyRange KeyRange; + + struct TLess { + TConstArrayRef<NScheme::TTypeId> Schema; + + TLess(const TVector<NScheme::TTypeId>& schema) + : Schema(schema) + { + } + + bool operator()(const TPQPartitionInfo& lhs, const TPQPartitionInfo& rhs) const { + Y_VERIFY(lhs.KeyRange.ToBound || rhs.KeyRange.ToBound); + + if (!lhs.KeyRange.ToBound) { + return false; + } + + if (!rhs.KeyRange.ToBound) { + return true; + } + + Y_VERIFY(lhs.KeyRange.ToBound && rhs.KeyRange.ToBound); + + const int compares = CompareTypedCellVectors( + lhs.KeyRange.ToBound->GetCells().data(), + rhs.KeyRange.ToBound->GetCells().data(), + Schema.data(), Schema.size() + ); + + return (compares < 0); + } + + }; // TLess + + }; // TPQPartitionInfo + + struct TKeyDesc { + struct TPartitionInfo { + ui32 PartitionId; + ui64 ShardId; + TSerializedCellVec EndKeyPrefix; + // just a hint + static constexpr bool IsInclusive = false; + static constexpr bool IsPoint = false; + + explicit TPartitionInfo(const TPQPartitionInfo& info) + : PartitionId(info.PartitionId) + , ShardId(info.ShardId) + { + if (info.KeyRange.ToBound) { + EndKeyPrefix = *info.KeyRange.ToBound; + } + } + + }; // TPartitionInfo + + TVector<NScheme::TTypeId> Schema; + TVector<TPartitionInfo> Partitions; + + }; // TKeyDesc + + TStringBuf GetLogPrefix() const { + if (!LogPrefix) { + LogPrefix = TStringBuilder() + << "[CdcChangeSenderMain]" + << "[" << DataShard.TabletId << ":" << DataShard.Generation << "]" + << SelfId() /* contains brackets */ << " "; + } + + return LogPrefix.GetRef(); + } + + bool IsResolvingCdcStream() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveCdcStream); + } + + bool IsResolvingTopic() const { + return CurrentStateFunc() == static_cast<TReceiveFunc>(&TThis::StateResolveTopic); + } + + bool IsResolving() const override { + return IsResolvingCdcStream() + || IsResolvingTopic(); + } + + TStringBuf CurrentStateName() const { + if (IsResolvingCdcStream()) { + return "ResolveCdcStream"; + } else if (IsResolvingTopic()) { + return "ResolveTopic"; + } else { + return ""; + } + } + + void Retry() { + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); + } + + void LogCritAndRetry(const TString& error) { + LOG_C(error); + Retry(); + } + + void LogWarnAndRetry(const TString& error) { + LOG_W(error); + Retry(); + } + + template <typename CheckFunc, typename FailFunc, typename T, typename... Args> + bool Check(CheckFunc checkFunc, FailFunc failFunc, const T& subject, Args&&... args) { + return checkFunc(CurrentStateName(), subject, std::forward<Args>(args)..., std::bind(failFunc, this, std::placeholders::_1)); + } + + bool CheckNotEmpty(const TAutoPtr<TNavigate>& result) { + return Check(&TSchemeCacheHelpers::CheckNotEmpty<TNavigate>, &TThis::LogCritAndRetry, result); + } + + bool CheckEntriesCount(const TAutoPtr<TNavigate>& result, ui32 expected) { + return Check(&TSchemeCacheHelpers::CheckEntriesCount<TNavigate>, &TThis::LogCritAndRetry, result, expected); + } + + bool CheckTableId(const TNavigate::TEntry& entry, const TTableId& expected) { + return Check(&TSchemeCacheHelpers::CheckTableId<TNavigate::TEntry>, &TThis::LogCritAndRetry, entry, expected); + } + + bool CheckEntrySucceeded(const TNavigate::TEntry& entry) { + return Check(&TSchemeCacheHelpers::CheckEntrySucceeded<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry); + } + + bool CheckEntryKind(const TNavigate::TEntry& entry, TNavigate::EKind expected) { + return Check(&TSchemeCacheHelpers::CheckEntryKind<TNavigate::TEntry>, &TThis::LogWarnAndRetry, entry, expected); + } + + bool CheckNotEmpty(const TIntrusiveConstPtr<TNavigate::TPQGroupInfo>& pqInfo) { + if (pqInfo) { + return true; + } + + LogCritAndRetry(TStringBuilder() << "Empty pq info at '" << CurrentStateName() << "'"); + return false; + } + + static TVector<ui64> MakePartitionIds(const TVector<TKeyDesc::TPartitionInfo>& partitions) { + TVector<ui64> result(Reserve(partitions.size())); + + for (const auto& partition : partitions) { + result.push_back(partition.PartitionId); + } + + return result; + } + + /// ResolveCdcStream + + void ResolveCdcStream() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(PathId, TNavigate::OpList)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveCdcStream); + } + + STATEFN(StateResolveCdcStream) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleCdcStream); + sFunc(TEvents::TEvWakeup, ResolveCdcStream); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleCdcStream(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, PathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindCdcStream)) { + return; + } + + Y_VERIFY(entry.ListNodeEntry->Children.size() == 1); + const auto& topic = entry.ListNodeEntry->Children.at(0); + + Y_VERIFY(topic.Kind == TNavigate::KindTopic); + TopicPathId = topic.PathId; + + ResolveTopic(); + } + + /// ResolveTopic + + void ResolveTopic() { + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(TopicPathId, TNavigate::OpTopic)); + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveTopic); + } + + STATEFN(StateResolveTopic) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTopic); + sFunc(TEvents::TEvWakeup, ResolveTopic); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandleTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& result = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(result)) { + return; + } + + if (!CheckEntriesCount(result, 1)) { + return; + } + + const auto& entry = result->ResultSet.at(0); + + if (!CheckTableId(entry, TopicPathId)) { + return; + } + + if (!CheckEntrySucceeded(entry)) { + return; + } + + if (!CheckEntryKind(entry, TNavigate::KindTopic)) { + return; + } + + if (!CheckNotEmpty(entry.PQGroupInfo)) { + return; + } + + const auto& pqDesc = entry.PQGroupInfo->Description; + const auto& pqConfig = pqDesc.GetPQTabletConfig(); + + KeyDesc = MakeHolder<TKeyDesc>(); + PartitionToShard.clear(); + + KeyDesc->Schema.reserve(pqConfig.PartitionKeySchemaSize()); + for (const auto& keySchema : pqConfig.GetPartitionKeySchema()) { + KeyDesc->Schema.push_back(keySchema.GetTypeId()); + } + + TSet<TPQPartitionInfo, TPQPartitionInfo::TLess> partitions(KeyDesc->Schema); + THashSet<ui64> shards; + + for (const auto& partition : pqDesc.GetPartitions()) { + const auto partitionId = partition.GetPartitionId(); + const auto shardId = partition.GetTabletId(); + + PartitionToShard.emplace(partitionId, shardId); + + auto keyRange = TPartitionKeyRange::Parse(partition.GetKeyRange()); + Y_VERIFY(!keyRange.FromBound || keyRange.FromBound->GetCells().size() == KeyDesc->Schema.size()); + Y_VERIFY(!keyRange.ToBound || keyRange.ToBound->GetCells().size() == KeyDesc->Schema.size()); + + partitions.insert({partitionId, shardId, std::move(keyRange)}); + shards.insert(shardId); + } + + // used to validate + bool isFirst = true; + const TPQPartitionInfo* prev = nullptr; + + KeyDesc->Partitions.reserve(partitions.size()); + for (const auto& cur : partitions) { + if (isFirst) { + isFirst = false; + Y_VERIFY(!cur.KeyRange.FromBound.Defined()); + } else { + Y_VERIFY(cur.KeyRange.FromBound.Defined()); + Y_VERIFY(prev); + Y_VERIFY(prev->KeyRange.ToBound.Defined()); + // TODO: compare cells + } + + KeyDesc->Partitions.emplace_back(cur); + prev = &cur; + } + + if (prev) { + Y_VERIFY(!prev->KeyRange.ToBound.Defined()); + } + + CreateSenders(MakePartitionIds(KeyDesc->Partitions)); + Become(&TThis::StateMain); + } + + /// Main + + STATEFN(StateMain) { + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + + void Resolve() override { + ResolveTopic(); + } + + bool IsResolved() const override { + return KeyDesc && KeyDesc->Partitions; + } + + ui64 GetPartitionId(TConstArrayRef<TCell> key) const override { + Y_VERIFY(KeyDesc); + Y_VERIFY(KeyDesc->Partitions); + + const auto range = TTableRange(key); + Y_VERIFY(range.Point); + + TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( + KeyDesc->Partitions.begin(), KeyDesc->Partitions.end(), true, + [&](const TKeyDesc::TPartitionInfo& partition, bool) { + const int compares = CompareBorders<true, false>( + partition.EndKeyPrefix.GetCells(), range.From, + partition.IsInclusive || partition.IsPoint, + range.InclusiveFrom || range.Point, KeyDesc->Schema + ); + + return (compares < 0); + } + ); + + Y_VERIFY(it != KeyDesc->Partitions.end()); + return it->PartitionId; + } + + IActor* CreateSender(ui64 partitionId) override { + Y_VERIFY(PartitionToShard.contains(partitionId)); + const auto shardId = PartitionToShard.at(partitionId); + return new TCdcChangeSenderPartition(SelfId(), DataShard, partitionId, shardId); + } + + void Handle(TEvChangeExchange::TEvEnqueueRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + EnqueueRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchange::TEvRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + ProcessRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchange::TEvForgetRecords::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + ForgetRecords(std::move(ev->Get()->Records)); + } + + void Handle(TEvChangeExchangePrivate::TEvReady::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + OnReady(ev->Get()->PartitionId); + } + + void Handle(TEvChangeExchangePrivate::TEvGone::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + OnGone(ev->Get()->PartitionId); + } + + void Handle(TEvChangeExchange::TEvRemoveSender::TPtr& ev) { + LOG_D("Handle " << ev->Get()->ToString()); + Y_VERIFY(ev->Get()->PathId == PathId); + + RemoveRecords(); + PassAway(); + } + + void PassAway() override { + KillSenders(); + TActorBootstrapped::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::CHANGE_SENDER_CDC_ACTOR_MAIN; + } + + explicit TCdcChangeSenderMain(const TDataShardId& dataShard, const TPathId& streamPathId) + : TActorBootstrapped() + , TBaseChangeSender(this, this, dataShard, streamPathId) + { + } + + void Bootstrap() { + ResolveCdcStream(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvChangeExchange::TEvEnqueueRecords, Handle); + hFunc(TEvChangeExchange::TEvRecords, Handle); + hFunc(TEvChangeExchange::TEvForgetRecords, Handle); + hFunc(TEvChangeExchange::TEvRemoveSender, Handle); + hFunc(TEvChangeExchangePrivate::TEvReady, Handle); + hFunc(TEvChangeExchangePrivate::TEvGone, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + mutable TMaybe<TString> LogPrefix; + + TPathId TopicPathId; + THolder<TKeyDesc> KeyDesc; + THashMap<ui32, ui64> PartitionToShard; + +}; // TCdcChangeSenderMain + +IActor* CreateCdcStreamChangeSender(const TDataShardId& dataShard, const TPathId& streamPathId) { + return new TCdcChangeSenderMain(dataShard, streamPathId); +} + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/change_sender_common_ops.cpp b/ydb/core/tx/datashard/change_sender_common_ops.cpp index 4d869ae9278..f7d4456da40 100644 --- a/ydb/core/tx/datashard/change_sender_common_ops.cpp +++ b/ydb/core/tx/datashard/change_sender_common_ops.cpp @@ -1,252 +1,252 @@ -#include "change_sender_common_ops.h" - -#include <util/generic/size_literals.h> - -namespace NKikimr { -namespace NDataShard { - -void TBaseChangeSender::CreateSenders(const TVector<ui64>& partitionIds) { - THashMap<ui64, TSender> senders; - - for (const auto& partitionId : partitionIds) { - auto it = Senders.find(partitionId); - if (it != Senders.end()) { - senders.emplace(partitionId, std::move(it->second)); - Senders.erase(it); - } else { - Y_VERIFY(!senders.contains(partitionId)); - auto& sender = senders[partitionId]; - sender.ActorId = ActorOps->Register(CreateSender(partitionId)); - } - } - - for (const auto& [_, sender] : Senders) { - ActorOps->Send(sender.ActorId, new TEvents::TEvPoisonPill()); - } - - Senders = std::move(senders); - - SendRecords(); -} - -void TBaseChangeSender::KillSenders() { - for (const auto& [_, sender] : Senders) { - ActorOps->Send(sender.ActorId, new TEvents::TEvPoisonPill()); - } - - Senders.clear(); -} - -void TBaseChangeSender::EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) { - for (auto& record : records) { - Y_VERIFY_S(PathId == record.PathId, "Unexpected record's path id" - << ": expected# " << PathId - << ", got# " << record.PathId); - Enqueued.emplace(record.Order, record.BodySize); - } - - RequestRecords(); -} - -void TBaseChangeSender::RequestRecords() { - if (!Enqueued) { - return; - } - - auto it = Enqueued.begin(); - TVector<TRequestedRecord> records; - - while (it != Enqueued.end()) { - if (MemUsage && (MemUsage + it->BodySize) > MemLimit) { - break; - } - - MemUsage += it->BodySize; - - records.emplace_back(it->Order, it->BodySize); - PendingBody.emplace(it->Order, it->BodySize); - it = Enqueued.erase(it); - } - - if (!records) { - return; - } - - ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRequestRecords(std::move(records))); -} - -void TBaseChangeSender::ProcessRecords(TVector<TChangeRecord>&& records) { - for (auto& record : records) { - auto it = PendingBody.find(record.GetOrder()); - if (it == PendingBody.end()) { - continue; - } - - if (it->BodySize != record.GetBody().size()) { - MemUsage -= it->BodySize; - MemUsage += record.GetBody().size(); - } - - PendingSent.emplace(record.GetOrder(), std::move(record)); - PendingBody.erase(it); - } - - SendRecords(); -} - -void TBaseChangeSender::SendRecords() { - if (!Resolver->IsResolved()) { - return; - } - - if (!PendingSent) { - return; - } - - auto it = PendingSent.begin(); - THashMap<ui64, TVector<TChangeRecord>> forward; - bool needToResolve = false; - - while (it != PendingSent.end()) { - const ui64 partitionId = Resolver->GetPartitionId(it->second.GetKey()); - if (!Senders.contains(partitionId)) { - needToResolve = true; - ++it; - continue; - } - - const auto& sender = Senders.at(partitionId); - if (!sender.Ready) { - ++it; - continue; - } - - MemUsage -= it->second.GetBody().size(); - - forward[partitionId].push_back(std::move(it->second)); - it = PendingSent.erase(it); - } - - for (auto& [partitionId, records] : forward) { - Y_VERIFY(Senders.contains(partitionId)); - auto& sender = Senders.at(partitionId); - - Y_VERIFY(sender.Ready); - sender.Ready = false; - - sender.Pending.reserve(records.size()); - for (const auto& record : records) { - sender.Pending.emplace_back(record.GetOrder(), record.GetBody().size()); - } - - ActorOps->Send(sender.ActorId, new TEvChangeExchange::TEvRecords(std::move(records))); - } - - if (needToResolve && !Resolver->IsResolving()) { - Resolver->Resolve(); - } - - RequestRecords(); -} - -void TBaseChangeSender::ForgetRecords(TVector<ui64>&& records) { - for (const auto& record : records) { - auto it = PendingBody.find(record); - if (it == PendingBody.end()) { - continue; - } - - MemUsage -= it->BodySize; - PendingBody.erase(it); - } - - RequestRecords(); -} - -void TBaseChangeSender::OnReady(ui64 partitionId) { - auto it = Senders.find(partitionId); - if (it == Senders.end()) { - return; - } - - auto& sender = it->second; - sender.Ready = true; - - if (sender.Pending) { - TVector<ui64> remove(Reserve(sender.Pending.size())); - for (const auto& record : sender.Pending) { - remove.push_back(record.Order); - } - - ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); - sender.Pending.clear(); - } - - SendRecords(); -} - -void TBaseChangeSender::OnGone(ui64 partitionId) { - auto it = Senders.find(partitionId); - if (it == Senders.end()) { - return; - } - - const auto& sender = it->second; - if (sender.Pending) { - Enqueued.insert(sender.Pending.begin(), sender.Pending.end()); - RequestRecords(); - } - - Senders.erase(it); - - if (Resolver->IsResolving()) { - return; - } - - Resolver->Resolve(); -} - -void TBaseChangeSender::RemoveRecords() { - ui64 pendingStatus = 0; - for (const auto& [_, sender] : Senders) { - pendingStatus += sender.Pending.size(); - } - - TVector<ui64> remove(Reserve(Enqueued.size() + PendingBody.size() + PendingSent.size() + pendingStatus)); - - for (const auto& record : Enqueued) { - remove.push_back(record.Order); - } - - for (const auto& record : PendingBody) { - remove.push_back(record.Order); - } - - for (const auto& [order, _] : PendingSent) { - remove.push_back(order); - } - - for (const auto& [_, sender] : Senders) { - for (const auto& record : sender.Pending) { - remove.push_back(record.Order); - } - } - - if (remove) { - ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); - } -} - -TBaseChangeSender::TBaseChangeSender(IActorOps* actorOps, IChangeSenderResolver* resolver, - const TDataShardId& dataShard, const TPathId& pathId) - : ActorOps(actorOps) - , Resolver(resolver) - , DataShard(dataShard) - , PathId(pathId) - , MemLimit(192_KB) - , MemUsage(0) -{ -} - -} // NDataShard -} // NKikimr +#include "change_sender_common_ops.h" + +#include <util/generic/size_literals.h> + +namespace NKikimr { +namespace NDataShard { + +void TBaseChangeSender::CreateSenders(const TVector<ui64>& partitionIds) { + THashMap<ui64, TSender> senders; + + for (const auto& partitionId : partitionIds) { + auto it = Senders.find(partitionId); + if (it != Senders.end()) { + senders.emplace(partitionId, std::move(it->second)); + Senders.erase(it); + } else { + Y_VERIFY(!senders.contains(partitionId)); + auto& sender = senders[partitionId]; + sender.ActorId = ActorOps->Register(CreateSender(partitionId)); + } + } + + for (const auto& [_, sender] : Senders) { + ActorOps->Send(sender.ActorId, new TEvents::TEvPoisonPill()); + } + + Senders = std::move(senders); + + SendRecords(); +} + +void TBaseChangeSender::KillSenders() { + for (const auto& [_, sender] : Senders) { + ActorOps->Send(sender.ActorId, new TEvents::TEvPoisonPill()); + } + + Senders.clear(); +} + +void TBaseChangeSender::EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) { + for (auto& record : records) { + Y_VERIFY_S(PathId == record.PathId, "Unexpected record's path id" + << ": expected# " << PathId + << ", got# " << record.PathId); + Enqueued.emplace(record.Order, record.BodySize); + } + + RequestRecords(); +} + +void TBaseChangeSender::RequestRecords() { + if (!Enqueued) { + return; + } + + auto it = Enqueued.begin(); + TVector<TRequestedRecord> records; + + while (it != Enqueued.end()) { + if (MemUsage && (MemUsage + it->BodySize) > MemLimit) { + break; + } + + MemUsage += it->BodySize; + + records.emplace_back(it->Order, it->BodySize); + PendingBody.emplace(it->Order, it->BodySize); + it = Enqueued.erase(it); + } + + if (!records) { + return; + } + + ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRequestRecords(std::move(records))); +} + +void TBaseChangeSender::ProcessRecords(TVector<TChangeRecord>&& records) { + for (auto& record : records) { + auto it = PendingBody.find(record.GetOrder()); + if (it == PendingBody.end()) { + continue; + } + + if (it->BodySize != record.GetBody().size()) { + MemUsage -= it->BodySize; + MemUsage += record.GetBody().size(); + } + + PendingSent.emplace(record.GetOrder(), std::move(record)); + PendingBody.erase(it); + } + + SendRecords(); +} + +void TBaseChangeSender::SendRecords() { + if (!Resolver->IsResolved()) { + return; + } + + if (!PendingSent) { + return; + } + + auto it = PendingSent.begin(); + THashMap<ui64, TVector<TChangeRecord>> forward; + bool needToResolve = false; + + while (it != PendingSent.end()) { + const ui64 partitionId = Resolver->GetPartitionId(it->second.GetKey()); + if (!Senders.contains(partitionId)) { + needToResolve = true; + ++it; + continue; + } + + const auto& sender = Senders.at(partitionId); + if (!sender.Ready) { + ++it; + continue; + } + + MemUsage -= it->second.GetBody().size(); + + forward[partitionId].push_back(std::move(it->second)); + it = PendingSent.erase(it); + } + + for (auto& [partitionId, records] : forward) { + Y_VERIFY(Senders.contains(partitionId)); + auto& sender = Senders.at(partitionId); + + Y_VERIFY(sender.Ready); + sender.Ready = false; + + sender.Pending.reserve(records.size()); + for (const auto& record : records) { + sender.Pending.emplace_back(record.GetOrder(), record.GetBody().size()); + } + + ActorOps->Send(sender.ActorId, new TEvChangeExchange::TEvRecords(std::move(records))); + } + + if (needToResolve && !Resolver->IsResolving()) { + Resolver->Resolve(); + } + + RequestRecords(); +} + +void TBaseChangeSender::ForgetRecords(TVector<ui64>&& records) { + for (const auto& record : records) { + auto it = PendingBody.find(record); + if (it == PendingBody.end()) { + continue; + } + + MemUsage -= it->BodySize; + PendingBody.erase(it); + } + + RequestRecords(); +} + +void TBaseChangeSender::OnReady(ui64 partitionId) { + auto it = Senders.find(partitionId); + if (it == Senders.end()) { + return; + } + + auto& sender = it->second; + sender.Ready = true; + + if (sender.Pending) { + TVector<ui64> remove(Reserve(sender.Pending.size())); + for (const auto& record : sender.Pending) { + remove.push_back(record.Order); + } + + ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); + sender.Pending.clear(); + } + + SendRecords(); +} + +void TBaseChangeSender::OnGone(ui64 partitionId) { + auto it = Senders.find(partitionId); + if (it == Senders.end()) { + return; + } + + const auto& sender = it->second; + if (sender.Pending) { + Enqueued.insert(sender.Pending.begin(), sender.Pending.end()); + RequestRecords(); + } + + Senders.erase(it); + + if (Resolver->IsResolving()) { + return; + } + + Resolver->Resolve(); +} + +void TBaseChangeSender::RemoveRecords() { + ui64 pendingStatus = 0; + for (const auto& [_, sender] : Senders) { + pendingStatus += sender.Pending.size(); + } + + TVector<ui64> remove(Reserve(Enqueued.size() + PendingBody.size() + PendingSent.size() + pendingStatus)); + + for (const auto& record : Enqueued) { + remove.push_back(record.Order); + } + + for (const auto& record : PendingBody) { + remove.push_back(record.Order); + } + + for (const auto& [order, _] : PendingSent) { + remove.push_back(order); + } + + for (const auto& [_, sender] : Senders) { + for (const auto& record : sender.Pending) { + remove.push_back(record.Order); + } + } + + if (remove) { + ActorOps->Send(DataShard.ActorId, new TEvChangeExchange::TEvRemoveRecords(std::move(remove))); + } +} + +TBaseChangeSender::TBaseChangeSender(IActorOps* actorOps, IChangeSenderResolver* resolver, + const TDataShardId& dataShard, const TPathId& pathId) + : ActorOps(actorOps) + , Resolver(resolver) + , DataShard(dataShard) + , PathId(pathId) + , MemLimit(192_KB) + , MemUsage(0) +{ +} + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/change_sender_common_ops.h b/ydb/core/tx/datashard/change_sender_common_ops.h index a04816f30d7..2ba88be3f22 100644 --- a/ydb/core/tx/datashard/change_sender_common_ops.h +++ b/ydb/core/tx/datashard/change_sender_common_ops.h @@ -1,222 +1,222 @@ -#pragma once - -#include "change_exchange.h" -#include "change_exchange_helpers.h" - +#pragma once + +#include "change_exchange.h" +#include "change_exchange_helpers.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/set.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NDataShard { - -struct TEvChangeExchangePrivate { - enum EEv { - EvReady = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvGone, - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)); - - template <typename TEv, ui32 TEventType> - struct TEvWithPartitionId: public TEventLocal<TEv, TEventType> { - ui64 PartitionId; - - explicit TEvWithPartitionId(ui64 partiionId) - : PartitionId(partiionId) - { - } - - TString ToString() const override { - return TStringBuilder() << TEventLocal<TEv, TEventType>::ToStringHeader() << " {" - << " PartitionId: " << PartitionId - << " }"; - } - }; - - struct TEvReady: public TEvWithPartitionId<TEvReady, EvReady> { - using TEvWithPartitionId::TEvWithPartitionId; - }; - - struct TEvGone: public TEvWithPartitionId<TEvGone, EvGone> { - using TEvWithPartitionId::TEvWithPartitionId; - }; - -}; // TEvChangeExchangePrivate - -class IChangeSender { -public: - virtual ~IChangeSender() = default; - - virtual void CreateSenders(const TVector<ui64>& partitionIds) = 0; - virtual void KillSenders() = 0; - virtual IActor* CreateSender(ui64 partitionId) = 0; - virtual void RemoveRecords() = 0; - - virtual void EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) = 0; - virtual void ProcessRecords(TVector<TChangeRecord>&& records) = 0; - virtual void ForgetRecords(TVector<ui64>&& records) = 0; - virtual void OnReady(ui64 partitionId) = 0; - virtual void OnGone(ui64 partitionId) = 0; -}; - -class IChangeSenderResolver { -public: - virtual ~IChangeSenderResolver() = default; - - virtual void Resolve() = 0; - virtual bool IsResolving() const = 0; - virtual bool IsResolved() const = 0; - virtual ui64 GetPartitionId(TConstArrayRef<TCell> key) const = 0; -}; - -class TBaseChangeSender: public IChangeSender { - using TEnqueuedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; - using TRequestedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; - - struct TSender { - TActorId ActorId; - bool Ready = false; - TVector<TEnqueuedRecord> Pending; - }; - - void RequestRecords(); - void SendRecords(); - -protected: - void CreateSenders(const TVector<ui64>& partitionIds) override; - void KillSenders() override; - void RemoveRecords() override; - - void EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) override; - void ProcessRecords(TVector<TChangeRecord>&& records) override; - void ForgetRecords(TVector<ui64>&& records) override; - void OnReady(ui64 partitionId) override; - void OnGone(ui64 partitionId) override; - - explicit TBaseChangeSender(IActorOps* actorOps, IChangeSenderResolver* resolver, - const TDataShardId& dataShard, const TPathId& pathId); - -private: - IActorOps* const ActorOps; - IChangeSenderResolver* const Resolver; - -protected: - const TDataShardId DataShard; - const TPathId PathId; - -private: - const ui64 MemLimit; - ui64 MemUsage; - - THashMap<ui64, TSender> Senders; // ui64 is partition id - TSet<TEnqueuedRecord> Enqueued; - TSet<TRequestedRecord> PendingBody; - TMap<ui64, TChangeRecord> PendingSent; // ui64 is order - -}; // TBaseChangeSender - -struct TSchemeCacheHelpers { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TResolve = NSchemeCache::TSchemeCacheRequest; - using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; - using TCheckFailFunc = std::function<void(const TString&)>; - - inline static TNavigate::TEntry MakeNavigateEntry(const TTableId& tableId, TNavigate::EOp op) { - TNavigate::TEntry entry; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - entry.TableId = tableId; - entry.Operation = op; - entry.ShowPrivatePath = true; - return entry; - } - - template <typename T> - static bool CheckNotEmpty(const TStringBuf marker, const TAutoPtr<T>& result, TCheckFailFunc onFailure) { - if (result) { - return true; - } - - onFailure(TStringBuilder() << "Empty result at '" << marker << "'"); - return false; - } - - template <typename T> - static bool CheckEntriesCount(const TStringBuf marker, const TAutoPtr<T>& result, ui32 expected, TCheckFailFunc onFailure) { - if (result->ResultSet.size() == expected) { - return true; - } - - onFailure(TStringBuilder() << "Unexpected entries count at '" << marker << "'" - << ": expected# " << expected - << ", got# " << result->ResultSet.size() - << ", result# " << result->ToString(*AppData()->TypeRegistry)); - return false; - } - - inline static const TTableId& GetTableId(const TNavigate::TEntry& entry) { - return entry.TableId; - } - - inline static const TTableId& GetTableId(const TResolve::TEntry& entry) { - return entry.KeyDescription->TableId; - } - - template <typename T> - static bool CheckTableId(const TStringBuf marker, const T& entry, const TTableId& expected, TCheckFailFunc onFailure) { - if (GetTableId(entry).HasSamePath(expected)) { - return true; - } - - onFailure(TStringBuilder() << "Unexpected table id at '" << marker << "'" - << ": expected# " << expected - << ", got# " << GetTableId(entry) - << ", entry# " << entry.ToString()); - return false; - } - - inline static bool IsSucceeded(TNavigate::EStatus status) { - return status == TNavigate::EStatus::Ok; - } - - inline static bool IsSucceeded(TResolve::EStatus status) { - return status == TResolve::EStatus::OkData; - } - - template <typename T> - static bool CheckEntrySucceeded(const TStringBuf marker, const T& entry, TCheckFailFunc onFailure) { - if (IsSucceeded(entry.Status)) { - return true; - } - - onFailure(TStringBuilder() << "Failed entry at '" << marker << "'" - << ": entry# " << entry.ToString()); - return false; - } - - template <typename T> - static bool CheckEntryKind(const TStringBuf marker, const T& entry, TNavigate::EKind expected, TCheckFailFunc onFailure) { - if (entry.Kind == expected) { - return true; - } - - onFailure(TStringBuilder() << "Unexpected entry kind at '" << marker << "'" - << ", expected# " << static_cast<ui32>(expected) - << ", got# " << static_cast<ui32>(entry.Kind) - << ", entry# " << entry.ToString()); - return false; - } - -}; // TSchemeCacheHelpers - -} // NDataShard -} // NKikimr + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NDataShard { + +struct TEvChangeExchangePrivate { + enum EEv { + EvReady = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvGone, + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)); + + template <typename TEv, ui32 TEventType> + struct TEvWithPartitionId: public TEventLocal<TEv, TEventType> { + ui64 PartitionId; + + explicit TEvWithPartitionId(ui64 partiionId) + : PartitionId(partiionId) + { + } + + TString ToString() const override { + return TStringBuilder() << TEventLocal<TEv, TEventType>::ToStringHeader() << " {" + << " PartitionId: " << PartitionId + << " }"; + } + }; + + struct TEvReady: public TEvWithPartitionId<TEvReady, EvReady> { + using TEvWithPartitionId::TEvWithPartitionId; + }; + + struct TEvGone: public TEvWithPartitionId<TEvGone, EvGone> { + using TEvWithPartitionId::TEvWithPartitionId; + }; + +}; // TEvChangeExchangePrivate + +class IChangeSender { +public: + virtual ~IChangeSender() = default; + + virtual void CreateSenders(const TVector<ui64>& partitionIds) = 0; + virtual void KillSenders() = 0; + virtual IActor* CreateSender(ui64 partitionId) = 0; + virtual void RemoveRecords() = 0; + + virtual void EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) = 0; + virtual void ProcessRecords(TVector<TChangeRecord>&& records) = 0; + virtual void ForgetRecords(TVector<ui64>&& records) = 0; + virtual void OnReady(ui64 partitionId) = 0; + virtual void OnGone(ui64 partitionId) = 0; +}; + +class IChangeSenderResolver { +public: + virtual ~IChangeSenderResolver() = default; + + virtual void Resolve() = 0; + virtual bool IsResolving() const = 0; + virtual bool IsResolved() const = 0; + virtual ui64 GetPartitionId(TConstArrayRef<TCell> key) const = 0; +}; + +class TBaseChangeSender: public IChangeSender { + using TEnqueuedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; + using TRequestedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; + + struct TSender { + TActorId ActorId; + bool Ready = false; + TVector<TEnqueuedRecord> Pending; + }; + + void RequestRecords(); + void SendRecords(); + +protected: + void CreateSenders(const TVector<ui64>& partitionIds) override; + void KillSenders() override; + void RemoveRecords() override; + + void EnqueueRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) override; + void ProcessRecords(TVector<TChangeRecord>&& records) override; + void ForgetRecords(TVector<ui64>&& records) override; + void OnReady(ui64 partitionId) override; + void OnGone(ui64 partitionId) override; + + explicit TBaseChangeSender(IActorOps* actorOps, IChangeSenderResolver* resolver, + const TDataShardId& dataShard, const TPathId& pathId); + +private: + IActorOps* const ActorOps; + IChangeSenderResolver* const Resolver; + +protected: + const TDataShardId DataShard; + const TPathId PathId; + +private: + const ui64 MemLimit; + ui64 MemUsage; + + THashMap<ui64, TSender> Senders; // ui64 is partition id + TSet<TEnqueuedRecord> Enqueued; + TSet<TRequestedRecord> PendingBody; + TMap<ui64, TChangeRecord> PendingSent; // ui64 is order + +}; // TBaseChangeSender + +struct TSchemeCacheHelpers { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TResolve = NSchemeCache::TSchemeCacheRequest; + using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; + using TCheckFailFunc = std::function<void(const TString&)>; + + inline static TNavigate::TEntry MakeNavigateEntry(const TTableId& tableId, TNavigate::EOp op) { + TNavigate::TEntry entry; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.TableId = tableId; + entry.Operation = op; + entry.ShowPrivatePath = true; + return entry; + } + + template <typename T> + static bool CheckNotEmpty(const TStringBuf marker, const TAutoPtr<T>& result, TCheckFailFunc onFailure) { + if (result) { + return true; + } + + onFailure(TStringBuilder() << "Empty result at '" << marker << "'"); + return false; + } + + template <typename T> + static bool CheckEntriesCount(const TStringBuf marker, const TAutoPtr<T>& result, ui32 expected, TCheckFailFunc onFailure) { + if (result->ResultSet.size() == expected) { + return true; + } + + onFailure(TStringBuilder() << "Unexpected entries count at '" << marker << "'" + << ": expected# " << expected + << ", got# " << result->ResultSet.size() + << ", result# " << result->ToString(*AppData()->TypeRegistry)); + return false; + } + + inline static const TTableId& GetTableId(const TNavigate::TEntry& entry) { + return entry.TableId; + } + + inline static const TTableId& GetTableId(const TResolve::TEntry& entry) { + return entry.KeyDescription->TableId; + } + + template <typename T> + static bool CheckTableId(const TStringBuf marker, const T& entry, const TTableId& expected, TCheckFailFunc onFailure) { + if (GetTableId(entry).HasSamePath(expected)) { + return true; + } + + onFailure(TStringBuilder() << "Unexpected table id at '" << marker << "'" + << ": expected# " << expected + << ", got# " << GetTableId(entry) + << ", entry# " << entry.ToString()); + return false; + } + + inline static bool IsSucceeded(TNavigate::EStatus status) { + return status == TNavigate::EStatus::Ok; + } + + inline static bool IsSucceeded(TResolve::EStatus status) { + return status == TResolve::EStatus::OkData; + } + + template <typename T> + static bool CheckEntrySucceeded(const TStringBuf marker, const T& entry, TCheckFailFunc onFailure) { + if (IsSucceeded(entry.Status)) { + return true; + } + + onFailure(TStringBuilder() << "Failed entry at '" << marker << "'" + << ": entry# " << entry.ToString()); + return false; + } + + template <typename T> + static bool CheckEntryKind(const TStringBuf marker, const T& entry, TNavigate::EKind expected, TCheckFailFunc onFailure) { + if (entry.Kind == expected) { + return true; + } + + onFailure(TStringBuilder() << "Unexpected entry kind at '" << marker << "'" + << ", expected# " << static_cast<ui32>(expected) + << ", got# " << static_cast<ui32>(entry.Kind) + << ", entry# " << entry.ToString()); + return false; + } + +}; // TSchemeCacheHelpers + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp b/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp index 60c398d5513..956f6cccba5 100644 --- a/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp @@ -1,126 +1,126 @@ -#include "datashard_active_transaction.h" -#include "datashard_direct_erase.h" -#include "datashard_distributed_erase.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -#include <util/generic/bitmap.h> -#include <util/string/builder.h> - -namespace NKikimr { +#include "datashard_active_transaction.h" +#include "datashard_direct_erase.h" +#include "datashard_distributed_erase.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +#include <util/generic/bitmap.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -class TCheckDistributedEraseTxUnit : public TExecutionUnit { -public: + +class TCheckDistributedEraseTxUnit : public TExecutionUnit { +public: TCheckDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::CheckDistributedEraseTx, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override { - Y_VERIFY(op->IsDistributedEraseTx()); - Y_VERIFY(!op->IsAborted()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - if (CheckRejectDataTx(op, ctx)) { - op->Abort(EExecutionUnitKind::FinishPropose); - return EExecutionStatus::Executed; - } - - const auto& eraseTx = tx->GetDistributedEraseTx(); - const auto& request = eraseTx->GetRequest(); - - auto buildUnsuccessfulResult = [&]( - const TString& reason, - NKikimrTxDataShard::TEvProposeTransactionResult::EStatus status = NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST, - NKikimrTxDataShard::TError::EKind kind = NKikimrTxDataShard::TError::BAD_ARGUMENT) { - - BuildResult(op, status)->AddError(kind, reason); - op->Abort(EExecutionUnitKind::FinishPropose); - return EExecutionStatus::Executed; - }; - - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus status; - TString error; - if (!TDirectTxErase::CheckRequest(&DataShard, request, status, error)) { - return buildUnsuccessfulResult(error); - } - - const size_t count = (size_t)eraseTx->HasDependents() + (size_t)eraseTx->HasDependencies(); - if (count != 1) { - return buildUnsuccessfulResult("Distributed erase tx can only have dependents or dependencies, not both"); - } - - for (const auto& dependency : eraseTx->GetDependencies()) { - auto presentRows = DeserializeBitMap<TDynBitMap>(dependency.GetPresentRows()); - if (request.KeyColumnsSize() == presentRows.Count()) { - continue; - } - - return buildUnsuccessfulResult(TStringBuilder() << "Present rows count mismatch" - << ": got " << presentRows.Count() - << ", expected " << request.KeyColumnsSize() - ); - } - - // checked at CheckedExecute stage - Y_VERIFY(DataShard.GetUserTables().contains(request.GetTableId())); - const TUserTable& tableInfo = *DataShard.GetUserTables().at(request.GetTableId()); - - for (const auto columnId : eraseTx->GetIndexColumnIds()) { - if (tableInfo.Columns.contains(columnId)) { - continue; - } - - return buildUnsuccessfulResult(TStringBuilder() << "Unknown index column id " << columnId); - } - - for (const auto& serializedCells : eraseTx->GetIndexColumns()) { - TSerializedCellVec indexCells; - if (!TSerializedCellVec::TryParse(serializedCells, indexCells)) { - return buildUnsuccessfulResult("Cannot parse index column value"); - } - - if (indexCells.GetCells().size() != static_cast<ui32>(eraseTx->GetIndexColumnIds().size())) { - return buildUnsuccessfulResult("Cell count doesn't match row scheme"); - } - } - - if (!Pipeline.AssignPlanInterval(op)) { - const TString err = TStringBuilder() << "Can't propose" - << " tx " << op->GetTxId() - << " at blocked shard " << DataShard.TabletID(); - - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); - return buildUnsuccessfulResult( - err, - NKikimrTxDataShard::TEvProposeTransactionResult::ERROR, - NKikimrTxDataShard::TError::SHARD_IS_BLOCKED - ); - } - - BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Prepared" - << " " << op->GetKind() - << " transaction txId " << op->GetTxId() - << " at tablet " << DataShard.TabletID()); - return EExecutionStatus::Executed; - } - - void Complete(TOperation::TPtr, const TActorContext&) override { - } -}; - + : TExecutionUnit(EExecutionUnitKind::CheckDistributedEraseTx, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override { + Y_VERIFY(op->IsDistributedEraseTx()); + Y_VERIFY(!op->IsAborted()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + if (CheckRejectDataTx(op, ctx)) { + op->Abort(EExecutionUnitKind::FinishPropose); + return EExecutionStatus::Executed; + } + + const auto& eraseTx = tx->GetDistributedEraseTx(); + const auto& request = eraseTx->GetRequest(); + + auto buildUnsuccessfulResult = [&]( + const TString& reason, + NKikimrTxDataShard::TEvProposeTransactionResult::EStatus status = NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST, + NKikimrTxDataShard::TError::EKind kind = NKikimrTxDataShard::TError::BAD_ARGUMENT) { + + BuildResult(op, status)->AddError(kind, reason); + op->Abort(EExecutionUnitKind::FinishPropose); + return EExecutionStatus::Executed; + }; + + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus status; + TString error; + if (!TDirectTxErase::CheckRequest(&DataShard, request, status, error)) { + return buildUnsuccessfulResult(error); + } + + const size_t count = (size_t)eraseTx->HasDependents() + (size_t)eraseTx->HasDependencies(); + if (count != 1) { + return buildUnsuccessfulResult("Distributed erase tx can only have dependents or dependencies, not both"); + } + + for (const auto& dependency : eraseTx->GetDependencies()) { + auto presentRows = DeserializeBitMap<TDynBitMap>(dependency.GetPresentRows()); + if (request.KeyColumnsSize() == presentRows.Count()) { + continue; + } + + return buildUnsuccessfulResult(TStringBuilder() << "Present rows count mismatch" + << ": got " << presentRows.Count() + << ", expected " << request.KeyColumnsSize() + ); + } + + // checked at CheckedExecute stage + Y_VERIFY(DataShard.GetUserTables().contains(request.GetTableId())); + const TUserTable& tableInfo = *DataShard.GetUserTables().at(request.GetTableId()); + + for (const auto columnId : eraseTx->GetIndexColumnIds()) { + if (tableInfo.Columns.contains(columnId)) { + continue; + } + + return buildUnsuccessfulResult(TStringBuilder() << "Unknown index column id " << columnId); + } + + for (const auto& serializedCells : eraseTx->GetIndexColumns()) { + TSerializedCellVec indexCells; + if (!TSerializedCellVec::TryParse(serializedCells, indexCells)) { + return buildUnsuccessfulResult("Cannot parse index column value"); + } + + if (indexCells.GetCells().size() != static_cast<ui32>(eraseTx->GetIndexColumnIds().size())) { + return buildUnsuccessfulResult("Cell count doesn't match row scheme"); + } + } + + if (!Pipeline.AssignPlanInterval(op)) { + const TString err = TStringBuilder() << "Can't propose" + << " tx " << op->GetTxId() + << " at blocked shard " << DataShard.TabletID(); + + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + return buildUnsuccessfulResult( + err, + NKikimrTxDataShard::TEvProposeTransactionResult::ERROR, + NKikimrTxDataShard::TError::SHARD_IS_BLOCKED + ); + } + + BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); + + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Prepared" + << " " << op->GetKind() + << " transaction txId " << op->GetTxId() + << " at tablet " << DataShard.TabletID()); + return EExecutionStatus::Executed; + } + + void Complete(TOperation::TPtr, const TActorContext&) override { + } +}; + THolder<TExecutionUnit> CreateCheckDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TCheckDistributedEraseTxUnit(self, pipeline)); -} - + return THolder(new TCheckDistributedEraseTxUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp index 90723f08adf..bbba1c6d95d 100644 --- a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp @@ -26,7 +26,7 @@ private: bool CheckDrop(TActiveTransaction *activeTx); bool CheckAlter(TActiveTransaction *activeTx); bool CheckBackup(TActiveTransaction *activeTx); - bool CheckRestore(TActiveTransaction *activeTx); + bool CheckRestore(TActiveTransaction *activeTx); bool CheckCopy(TActiveTransaction *activeTx); bool CheckCreatePersistentSnapshot(TActiveTransaction *activeTx); bool CheckDropPersistentSnapshot(TActiveTransaction *activeTx); @@ -34,20 +34,20 @@ private: bool CheckFinalizeBuildIndex(TActiveTransaction *activeTx); bool CheckDropIndexNotice(TActiveTransaction *activeTx); bool CheckMoveTable(TActiveTransaction *activeTx); - bool CheckCreateCdcStream(TActiveTransaction *activeTx); - bool CheckAlterCdcStream(TActiveTransaction *activeTx); - bool CheckDropCdcStream(TActiveTransaction *activeTx); + bool CheckCreateCdcStream(TActiveTransaction *activeTx); + bool CheckAlterCdcStream(TActiveTransaction *activeTx); + bool CheckDropCdcStream(TActiveTransaction *activeTx); bool CheckSchemaVersion(TActiveTransaction *activeTx, ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion); - - using TPipelineHasSmthFunc = std::function<bool(TPipeline* const)>; - bool HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker); - bool HasConflicts(TActiveTransaction *activeTx, const TStringBuf kind, const THashMap<TString, TPipelineHasSmthFunc>& checkers); - - template <typename T> bool HasPathId(TActiveTransaction *activeTx, const T &op, const TStringBuf kind); - template <typename T> TPathId GetPathId(const T &op) const; - template <typename T> bool CheckSchemaVersion(TActiveTransaction *activeTx, ui64 tableId, const T &op); - template <typename T> bool CheckSchemaVersion(TActiveTransaction *activeTx, const T &op); + + using TPipelineHasSmthFunc = std::function<bool(TPipeline* const)>; + bool HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker); + bool HasConflicts(TActiveTransaction *activeTx, const TStringBuf kind, const THashMap<TString, TPipelineHasSmthFunc>& checkers); + + template <typename T> bool HasPathId(TActiveTransaction *activeTx, const T &op, const TStringBuf kind); + template <typename T> TPathId GetPathId(const T &op) const; + template <typename T> bool CheckSchemaVersion(TActiveTransaction *activeTx, ui64 tableId, const T &op); + template <typename T> bool CheckSchemaVersion(TActiveTransaction *activeTx, const T &op); }; TCheckSchemeTxUnit::TCheckSchemeTxUnit(TDataShard &dataShard, @@ -231,138 +231,138 @@ bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, return true; } -bool TCheckSchemeTxUnit::HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker) { - if (!checker(&Pipeline)) { - return false; - } - - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring " << kind << " duplicate" - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); - BuildResult(activeTx)->SetSchemeTxDuplicate(Pipeline.CurrentSchemaTxId() == activeTx->GetTxId()); - - return true; -} - -bool TCheckSchemeTxUnit::HasConflicts(TActiveTransaction *activeTx, const TStringBuf kind, const THashMap<TString, TPipelineHasSmthFunc>& checkers) { - for (const auto& kv : checkers) { - const auto& conflicting = kv.first; - const auto& checker = kv.second; - - if (!checker(&Pipeline)) { - continue; - } - - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring " << kind << " during ongoing " << conflicting - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); - BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); - - return true; - } - - return false; -} - -template <typename T> -bool TCheckSchemeTxUnit::HasPathId(TActiveTransaction *activeTx, const T &op, const TStringBuf kind) { - if (op.HasPathId()) { - return true; - } - - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - kind << " description has no PathId" - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); - BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); - - return false; -} - -template <typename T> -TPathId TCheckSchemeTxUnit::GetPathId(const T &op) const { - auto pathId = TPathId(op.GetPathId().GetOwnerId(), op.GetPathId().GetLocalId()); - Y_VERIFY(DataShard.GetPathOwnerId() == pathId.OwnerId); - return pathId; -} - -template <typename T> -bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, ui64 tableId, const T &op) { - const auto tablePtr = DataShard.GetUserTables().FindPtr(tableId); - - Y_VERIFY(tablePtr); - const auto &table = **tablePtr; - - const auto current = table.GetTableSchemaVersion(); - const auto proposed = op.HasTableSchemaVersion() ? op.GetTableSchemaVersion() : 0; - - const auto res = CheckSchemaVersion(activeTx, proposed, current, current + 1); - Y_VERIFY_DEBUG(res, "Unexpected schema version mutation"); - - return true; -} - -template <typename T> -bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, const T &op) { - return CheckSchemaVersion(activeTx, GetPathId(op).LocalPathId, op); -} - +bool TCheckSchemeTxUnit::HasDuplicate(TActiveTransaction *activeTx, const TStringBuf kind, TPipelineHasSmthFunc checker) { + if (!checker(&Pipeline)) { + return false; + } + + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + "Ignoring " << kind << " duplicate" + << " at tablet " << DataShard.TabletID() + << " txId " << activeTx->GetTxId() + << " currentTxId " << Pipeline.CurrentSchemaTxId()); + BuildResult(activeTx)->SetSchemeTxDuplicate(Pipeline.CurrentSchemaTxId() == activeTx->GetTxId()); + + return true; +} + +bool TCheckSchemeTxUnit::HasConflicts(TActiveTransaction *activeTx, const TStringBuf kind, const THashMap<TString, TPipelineHasSmthFunc>& checkers) { + for (const auto& kv : checkers) { + const auto& conflicting = kv.first; + const auto& checker = kv.second; + + if (!checker(&Pipeline)) { + continue; + } + + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + "Ignoring " << kind << " during ongoing " << conflicting + << " at tablet " << DataShard.TabletID() + << " txId " << activeTx->GetTxId() + << " currentTxId " << Pipeline.CurrentSchemaTxId()); + BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); + + return true; + } + + return false; +} + +template <typename T> +bool TCheckSchemeTxUnit::HasPathId(TActiveTransaction *activeTx, const T &op, const TStringBuf kind) { + if (op.HasPathId()) { + return true; + } + + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, + kind << " description has no PathId" + << " at tablet " << DataShard.TabletID() + << " txId " << activeTx->GetTxId() + << " currentTxId " << Pipeline.CurrentSchemaTxId()); + BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); + + return false; +} + +template <typename T> +TPathId TCheckSchemeTxUnit::GetPathId(const T &op) const { + auto pathId = TPathId(op.GetPathId().GetOwnerId(), op.GetPathId().GetLocalId()); + Y_VERIFY(DataShard.GetPathOwnerId() == pathId.OwnerId); + return pathId; +} + +template <typename T> +bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, ui64 tableId, const T &op) { + const auto tablePtr = DataShard.GetUserTables().FindPtr(tableId); + + Y_VERIFY(tablePtr); + const auto &table = **tablePtr; + + const auto current = table.GetTableSchemaVersion(); + const auto proposed = op.HasTableSchemaVersion() ? op.GetTableSchemaVersion() : 0; + + const auto res = CheckSchemaVersion(activeTx, proposed, current, current + 1); + Y_VERIFY_DEBUG(res, "Unexpected schema version mutation"); + + return true; +} + +template <typename T> +bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, const T &op) { + return CheckSchemaVersion(activeTx, GetPathId(op).LocalPathId, op); +} + bool TCheckSchemeTxUnit::CheckSchemeTx(TActiveTransaction *activeTx) { const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); bool res = false; switch (activeTx->GetSchemeTxType()) { - case TSchemaOperation::ETypeCreate: + case TSchemaOperation::ETypeCreate: res = CheckCreate(activeTx); break; - case TSchemaOperation::ETypeDrop: + case TSchemaOperation::ETypeDrop: res = CheckDrop(activeTx); break; - case TSchemaOperation::ETypeAlter: + case TSchemaOperation::ETypeAlter: res = CheckAlter(activeTx); break; - case TSchemaOperation::ETypeBackup: + case TSchemaOperation::ETypeBackup: res = CheckBackup(activeTx); break; - case TSchemaOperation::ETypeRestore: - res = CheckRestore(activeTx); - break; - case TSchemaOperation::ETypeCopy: + case TSchemaOperation::ETypeRestore: + res = CheckRestore(activeTx); + break; + case TSchemaOperation::ETypeCopy: res = CheckCopy(activeTx); break; - case TSchemaOperation::ETypeCreatePersistentSnapshot: + case TSchemaOperation::ETypeCreatePersistentSnapshot: res = CheckCreatePersistentSnapshot(activeTx); break; - case TSchemaOperation::ETypeDropPersistentSnapshot: + case TSchemaOperation::ETypeDropPersistentSnapshot: res = CheckDropPersistentSnapshot(activeTx); break; - case TSchemaOperation::ETypeInitiateBuildIndex: + case TSchemaOperation::ETypeInitiateBuildIndex: res = CheckInitiateBuildIndex(activeTx); break; - case TSchemaOperation::ETypeFinalizeBuildIndex: + case TSchemaOperation::ETypeFinalizeBuildIndex: res = CheckFinalizeBuildIndex(activeTx); break; - case TSchemaOperation::ETypeDropIndexNotice: + case TSchemaOperation::ETypeDropIndexNotice: res = CheckDropIndexNotice(activeTx); break; case TSchemaOperation::ETypeMoveTable: res = CheckMoveTable(activeTx); break; - case TSchemaOperation::ETypeCreateCdcStream: - res = CheckCreateCdcStream(activeTx); + case TSchemaOperation::ETypeCreateCdcStream: + res = CheckCreateCdcStream(activeTx); + break; + case TSchemaOperation::ETypeAlterCdcStream: + res = CheckAlterCdcStream(activeTx); + break; + case TSchemaOperation::ETypeDropCdcStream: + res = CheckDropCdcStream(activeTx); break; - case TSchemaOperation::ETypeAlterCdcStream: - res = CheckAlterCdcStream(activeTx); - break; - case TSchemaOperation::ETypeDropCdcStream: - res = CheckDropCdcStream(activeTx); - break; default: LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, "Unknown scheme tx type detected at tablet " @@ -377,11 +377,11 @@ bool TCheckSchemeTxUnit::CheckSchemeTx(TActiveTransaction *activeTx) bool TCheckSchemeTxUnit::CheckCreate(TActiveTransaction *activeTx) { const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - if (HasDuplicate(activeTx, "Create", &TPipeline::HasCreate)) { + if (HasDuplicate(activeTx, "Create", &TPipeline::HasCreate)) { return false; } - const auto &create = tx.GetCreateTable(); + const auto &create = tx.GetCreateTable(); ui64 tableId = create.GetId_Deprecated(); if (create.HasPathId()) { Y_VERIFY(DataShard.GetPathOwnerId() == create.GetPathId().GetOwnerId() || DataShard.GetPathOwnerId() == INVALID_TABLET_ID); @@ -396,7 +396,7 @@ bool TCheckSchemeTxUnit::CheckCreate(TActiveTransaction *activeTx) { bool TCheckSchemeTxUnit::CheckDrop(TActiveTransaction *activeTx) { const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - if (HasDuplicate(activeTx, "Drop", &TPipeline::HasDrop)) { + if (HasDuplicate(activeTx, "Drop", &TPipeline::HasDrop)) { return false; } @@ -417,15 +417,15 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) Y_VERIFY(!Pipeline.HasDrop()); const TStringBuf kind = "Alter"; - if (HasDuplicate(activeTx, kind, &TPipeline::HasAlter)) { + if (HasDuplicate(activeTx, kind, &TPipeline::HasAlter)) { return false; } - const THashMap<TString, TPipelineHasSmthFunc> conflicting = { - {"Backup", &TPipeline::HasBackup}, - {"Restore", &TPipeline::HasRestore}, - }; - if (HasConflicts(activeTx, kind, conflicting)) { + const THashMap<TString, TPipelineHasSmthFunc> conflicting = { + {"Backup", &TPipeline::HasBackup}, + {"Restore", &TPipeline::HasRestore}, + }; + if (HasConflicts(activeTx, kind, conflicting)) { return false; } @@ -521,15 +521,15 @@ bool TCheckSchemeTxUnit::CheckBackup(TActiveTransaction *activeTx) Y_VERIFY(!Pipeline.HasDrop()); const TStringBuf kind = "Backup"; - if (HasDuplicate(activeTx, kind, &TPipeline::HasBackup)) { + if (HasDuplicate(activeTx, kind, &TPipeline::HasBackup)) { return false; } - const THashMap<TString, TPipelineHasSmthFunc> conflicting = { - {"Alter", &TPipeline::HasAlter}, - {"Restore", &TPipeline::HasRestore}, - }; - if (HasConflicts(activeTx, kind, conflicting)) { + const THashMap<TString, TPipelineHasSmthFunc> conflicting = { + {"Alter", &TPipeline::HasAlter}, + {"Restore", &TPipeline::HasRestore}, + }; + if (HasConflicts(activeTx, kind, conflicting)) { return false; } @@ -539,34 +539,34 @@ bool TCheckSchemeTxUnit::CheckBackup(TActiveTransaction *activeTx) return true; } -bool TCheckSchemeTxUnit::CheckRestore(TActiveTransaction *activeTx) -{ - const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - Y_VERIFY(!Pipeline.HasDrop()); - +bool TCheckSchemeTxUnit::CheckRestore(TActiveTransaction *activeTx) +{ + const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); + Y_VERIFY(!Pipeline.HasDrop()); + const TStringBuf kind = "Restore"; - if (HasDuplicate(activeTx, kind, &TPipeline::HasRestore)) { - return false; - } - - const THashMap<TString, TPipelineHasSmthFunc> conflicting = { - {"Alter", &TPipeline::HasAlter}, - {"Backup", &TPipeline::HasBackup}, - }; - if (HasConflicts(activeTx, kind, conflicting)) { - return false; - } - - const auto &restore = tx.GetRestore(); - Y_VERIFY(DataShard.GetUserTables().contains(restore.GetTableId())); - - return true; -} - + if (HasDuplicate(activeTx, kind, &TPipeline::HasRestore)) { + return false; + } + + const THashMap<TString, TPipelineHasSmthFunc> conflicting = { + {"Alter", &TPipeline::HasAlter}, + {"Backup", &TPipeline::HasBackup}, + }; + if (HasConflicts(activeTx, kind, conflicting)) { + return false; + } + + const auto &restore = tx.GetRestore(); + Y_VERIFY(DataShard.GetUserTables().contains(restore.GetTableId())); + + return true; +} + bool TCheckSchemeTxUnit::CheckCopy(TActiveTransaction *activeTx) { const NKikimrTxDataShard::TFlatSchemeTransaction &tx = activeTx->GetSchemeTx(); - if (HasDuplicate(activeTx, "Copy", &TPipeline::HasCopy)) { + if (HasDuplicate(activeTx, "Copy", &TPipeline::HasCopy)) { return false; } @@ -584,7 +584,7 @@ bool TCheckSchemeTxUnit::CheckCopy(TActiveTransaction *activeTx) { bool TCheckSchemeTxUnit::CheckCreatePersistentSnapshot(TActiveTransaction *activeTx) { const auto& tx = activeTx->GetSchemeTx(); - if (HasDuplicate(activeTx, "CreatePersistentSnapshot", &TPipeline::HasCreatePersistentSnapshot)) { + if (HasDuplicate(activeTx, "CreatePersistentSnapshot", &TPipeline::HasCreatePersistentSnapshot)) { return false; } @@ -596,7 +596,7 @@ bool TCheckSchemeTxUnit::CheckCreatePersistentSnapshot(TActiveTransaction *activ bool TCheckSchemeTxUnit::CheckDropPersistentSnapshot(TActiveTransaction *activeTx) { const auto& tx = activeTx->GetSchemeTx(); - if (HasDuplicate(activeTx, "DropPersistentSnapshot", &TPipeline::HasDropPersistentSnapshot)) { + if (HasDuplicate(activeTx, "DropPersistentSnapshot", &TPipeline::HasDropPersistentSnapshot)) { return false; } @@ -606,29 +606,29 @@ bool TCheckSchemeTxUnit::CheckDropPersistentSnapshot(TActiveTransaction *activeT } bool TCheckSchemeTxUnit::CheckInitiateBuildIndex(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "InitiateBuildIndex", &TPipeline::HasInitiateBuilIndex)) { + if (HasDuplicate(activeTx, "InitiateBuildIndex", &TPipeline::HasInitiateBuilIndex)) { return false; } - const auto &initiate = activeTx->GetSchemeTx().GetInitiateBuildIndex(); - if (!HasPathId(activeTx, initiate, "InitiateBuildIndex")) { - return false; + const auto &initiate = activeTx->GetSchemeTx().GetInitiateBuildIndex(); + if (!HasPathId(activeTx, initiate, "InitiateBuildIndex")) { + return false; } - return CheckSchemaVersion(activeTx, initiate); + return CheckSchemaVersion(activeTx, initiate); } bool TCheckSchemeTxUnit::CheckFinalizeBuildIndex(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "FinalizeBuildIndex", &TPipeline::HasFinalizeBuilIndex)) { + if (HasDuplicate(activeTx, "FinalizeBuildIndex", &TPipeline::HasFinalizeBuilIndex)) { return false; } - const auto &finalize = activeTx->GetSchemeTx().GetFinalizeBuildIndex(); - if (!HasPathId(activeTx, finalize, "FinalizeBuildIndex")) { - return false; + const auto &finalize = activeTx->GetSchemeTx().GetFinalizeBuildIndex(); + if (!HasPathId(activeTx, finalize, "FinalizeBuildIndex")) { + return false; } - const auto pathId = GetPathId(finalize); + const auto pathId = GetPathId(finalize); const auto snapshotKey = TSnapshotKey(pathId.OwnerId, pathId.LocalPathId, finalize.GetSnapshotStep(), finalize.GetSnapshotTxId()); if (DataShard.GetSnapshotManager().FindAvailable(snapshotKey) == nullptr) { @@ -639,78 +639,78 @@ bool TCheckSchemeTxUnit::CheckFinalizeBuildIndex(TActiveTransaction *activeTx) { << " pathId " << pathId << " snapshotKey " << snapshotKey); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); - return false; + return false; } - return CheckSchemaVersion(activeTx, pathId.LocalPathId, finalize); + return CheckSchemaVersion(activeTx, pathId.LocalPathId, finalize); } bool TCheckSchemeTxUnit::CheckDropIndexNotice(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "DropIndexNotice", &TPipeline::HasDropIndexNotice)) { + if (HasDuplicate(activeTx, "DropIndexNotice", &TPipeline::HasDropIndexNotice)) { return false; } - const auto ¬ice = activeTx->GetSchemeTx().GetDropIndexNotice(); - if (!HasPathId(activeTx, notice, "DropIndexNotice")) { - return false; + const auto ¬ice = activeTx->GetSchemeTx().GetDropIndexNotice(); + if (!HasPathId(activeTx, notice, "DropIndexNotice")) { + return false; } - return CheckSchemaVersion(activeTx, notice); + return CheckSchemaVersion(activeTx, notice); } bool TCheckSchemeTxUnit::CheckMoveTable(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "Move", &TPipeline::HasMove)) { + if (HasDuplicate(activeTx, "Move", &TPipeline::HasMove)) { return false; } - const auto &mv = activeTx->GetSchemeTx().GetMoveTable(); - if (!HasPathId(activeTx, mv, "Move")) { - return false; + const auto &mv = activeTx->GetSchemeTx().GetMoveTable(); + if (!HasPathId(activeTx, mv, "Move")) { + return false; + } + + return CheckSchemaVersion(activeTx, mv); +} + +bool TCheckSchemeTxUnit::CheckCreateCdcStream(TActiveTransaction *activeTx) { + if (HasDuplicate(activeTx, "CreateCdcStream", &TPipeline::HasCreateCdcStream)) { + return false; + } + + const auto ¬ice = activeTx->GetSchemeTx().GetCreateCdcStreamNotice(); + if (!HasPathId(activeTx, notice, "CreateCdcStream")) { + return false; + } + + return CheckSchemaVersion(activeTx, notice); +} + +bool TCheckSchemeTxUnit::CheckAlterCdcStream(TActiveTransaction *activeTx) { + if (HasDuplicate(activeTx, "AlterCdcStream", &TPipeline::HasAlterCdcStream)) { + return false; + } + + const auto ¬ice = activeTx->GetSchemeTx().GetAlterCdcStreamNotice(); + if (!HasPathId(activeTx, notice, "AlterCdcStream")) { + return false; + } + + return CheckSchemaVersion(activeTx, notice); +} + +bool TCheckSchemeTxUnit::CheckDropCdcStream(TActiveTransaction *activeTx) { + if (HasDuplicate(activeTx, "DropCdcStream", &TPipeline::HasDropCdcStream)) { + return false; + } + + const auto ¬ice = activeTx->GetSchemeTx().GetDropCdcStreamNotice(); + if (!HasPathId(activeTx, notice, "DropCdcStream")) { + return false; } - return CheckSchemaVersion(activeTx, mv); + return CheckSchemaVersion(activeTx, notice); } -bool TCheckSchemeTxUnit::CheckCreateCdcStream(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "CreateCdcStream", &TPipeline::HasCreateCdcStream)) { - return false; - } - - const auto ¬ice = activeTx->GetSchemeTx().GetCreateCdcStreamNotice(); - if (!HasPathId(activeTx, notice, "CreateCdcStream")) { - return false; - } - - return CheckSchemaVersion(activeTx, notice); -} - -bool TCheckSchemeTxUnit::CheckAlterCdcStream(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "AlterCdcStream", &TPipeline::HasAlterCdcStream)) { - return false; - } - - const auto ¬ice = activeTx->GetSchemeTx().GetAlterCdcStreamNotice(); - if (!HasPathId(activeTx, notice, "AlterCdcStream")) { - return false; - } - - return CheckSchemaVersion(activeTx, notice); -} - -bool TCheckSchemeTxUnit::CheckDropCdcStream(TActiveTransaction *activeTx) { - if (HasDuplicate(activeTx, "DropCdcStream", &TPipeline::HasDropCdcStream)) { - return false; - } - - const auto ¬ice = activeTx->GetSchemeTx().GetDropCdcStreamNotice(); - if (!HasPathId(activeTx, notice, "DropCdcStream")) { - return false; - } - - return CheckSchemaVersion(activeTx, notice); -} - void TCheckSchemeTxUnit::Complete(TOperation::TPtr, const TActorContext &) { diff --git a/ydb/core/tx/datashard/complete_data_tx_unit.cpp b/ydb/core/tx/datashard/complete_data_tx_unit.cpp index caa49125c2c..c086fa70119 100644 --- a/ydb/core/tx/datashard/complete_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/complete_data_tx_unit.cpp @@ -115,8 +115,8 @@ void TCompleteOperationUnit::Complete(TOperation::TPtr op, DataShard.SendDelayedAcks(ctx, op->DelayedAcks()); if (op->IsSchemeTx()) DataShard.NotifySchemeshard(ctx, op->GetTxId()); - - DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); + + DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); } THolder<TExecutionUnit> CreateCompleteOperationUnit(TDataShard &dataShard, diff --git a/ydb/core/tx/datashard/create_cdc_stream_unit.cpp b/ydb/core/tx/datashard/create_cdc_stream_unit.cpp index a55c4b76519..8f577f03b4c 100644 --- a/ydb/core/tx/datashard/create_cdc_stream_unit.cpp +++ b/ydb/core/tx/datashard/create_cdc_stream_unit.cpp @@ -1,79 +1,79 @@ -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -namespace NKikimr { +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +namespace NKikimr { namespace NDataShard { - -class TCreateCdcStreamUnit : public TExecutionUnit { - THolder<TEvChangeExchange::TEvAddSender> AddSender; - -public: + +class TCreateCdcStreamUnit : public TExecutionUnit { + THolder<TEvChangeExchange::TEvAddSender> AddSender; + +public: TCreateCdcStreamUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::CreateCdcStream, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { - Y_VERIFY(op->IsSchemeTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - auto& schemeTx = tx->GetSchemeTx(); - if (!schemeTx.HasCreateCdcStreamNotice()) { - return EExecutionStatus::Executed; - } - - const auto& params = schemeTx.GetCreateCdcStreamNotice(); - const auto& streamDesc = params.GetStreamDescription(); - - const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); - Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); - - const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); - Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); - - const auto version = params.GetTableSchemaVersion(); - Y_VERIFY(version); - - auto tableInfo = DataShard.AlterTableAddCdcStream(ctx, txc, pathId, version, streamDesc); - DataShard.AddUserTable(pathId, tableInfo); - - AddSender.Reset(new TEvChangeExchange::TEvAddSender( - pathId, TEvChangeExchange::ESenderType::CdcStream, streamPathId - )); - - /* TODO(ilnaz) - if (streamDesc.GetFormat() == NKikimrSchemeOp::ECdcStreamFormatJson) { - auto& manager = DataShard.GetSchemaSnapshotManager(); - - const auto key = TSchemaSnapshotKey(pathId.OwnerId, pathId.LocalPathId, version); - manager.AddSnapshot(txc.DB, key, TSchemaSnapshot(tableInfo, op->GetStep(), op->GetTxId())); - manager.AcquireReference(key); - } - */ - - BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); - op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); - - return EExecutionStatus::DelayCompleteNoMoreRestarts; - } - - void Complete(TOperation::TPtr, const TActorContext& ctx) override { - if (AddSender) { - ctx.Send(DataShard.GetChangeSender(), AddSender.Release()); - } - } -}; - + : TExecutionUnit(EExecutionUnitKind::CreateCdcStream, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { + Y_VERIFY(op->IsSchemeTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + auto& schemeTx = tx->GetSchemeTx(); + if (!schemeTx.HasCreateCdcStreamNotice()) { + return EExecutionStatus::Executed; + } + + const auto& params = schemeTx.GetCreateCdcStreamNotice(); + const auto& streamDesc = params.GetStreamDescription(); + + const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); + Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); + + const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); + Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); + + const auto version = params.GetTableSchemaVersion(); + Y_VERIFY(version); + + auto tableInfo = DataShard.AlterTableAddCdcStream(ctx, txc, pathId, version, streamDesc); + DataShard.AddUserTable(pathId, tableInfo); + + AddSender.Reset(new TEvChangeExchange::TEvAddSender( + pathId, TEvChangeExchange::ESenderType::CdcStream, streamPathId + )); + + /* TODO(ilnaz) + if (streamDesc.GetFormat() == NKikimrSchemeOp::ECdcStreamFormatJson) { + auto& manager = DataShard.GetSchemaSnapshotManager(); + + const auto key = TSchemaSnapshotKey(pathId.OwnerId, pathId.LocalPathId, version); + manager.AddSnapshot(txc.DB, key, TSchemaSnapshot(tableInfo, op->GetStep(), op->GetTxId())); + manager.AcquireReference(key); + } + */ + + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); + op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); + + return EExecutionStatus::DelayCompleteNoMoreRestarts; + } + + void Complete(TOperation::TPtr, const TActorContext& ctx) override { + if (AddSender) { + ctx.Send(DataShard.GetChangeSender(), AddSender.Release()); + } + } +}; + THolder<TExecutionUnit> CreateCreateCdcStreamUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TCreateCdcStreamUnit(self, pipeline)); -} - + return THolder(new TCreateCdcStreamUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/create_table_unit.cpp b/ydb/core/tx/datashard/create_table_unit.cpp index 39571fbf514..85f38d456a3 100644 --- a/ydb/core/tx/datashard/create_table_unit.cpp +++ b/ydb/core/tx/datashard/create_table_unit.cpp @@ -19,7 +19,7 @@ public: const TActorContext &ctx) override; private: - TVector<THolder<TEvChangeExchange::TEvAddSender>> AddSenders; + TVector<THolder<TEvChangeExchange::TEvAddSender>> AddSenders; }; TCreateTableUnit::TCreateTableUnit(TDataShard &dataShard, @@ -66,14 +66,14 @@ EExecutionStatus TCreateTableUnit::Execute(TOperation::TPtr op, TUserTable::TPtr info = DataShard.CreateUserTable(txc, schemeTx.GetCreateTable()); DataShard.AddUserTable(tableId, info); - for (const auto& [indexPathId, indexInfo] : info->Indexes) { + for (const auto& [indexPathId, indexInfo] : info->Indexes) { if (indexInfo.Type == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync) { - AddSenders.emplace_back(new TEvChangeExchange::TEvAddSender( - tableId, TEvChangeExchange::ESenderType::AsyncIndex, indexPathId - )); - } - } - + AddSenders.emplace_back(new TEvChangeExchange::TEvAddSender( + tableId, TEvChangeExchange::ESenderType::AsyncIndex, indexPathId + )); + } + } + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); @@ -83,16 +83,16 @@ EExecutionStatus TCreateTableUnit::Execute(TOperation::TPtr op, DataShard.CheckMvccStateChangeCanStart(ctx); // Recheck } - return EExecutionStatus::DelayCompleteNoMoreRestarts; + return EExecutionStatus::DelayCompleteNoMoreRestarts; } -void TCreateTableUnit::Complete(TOperation::TPtr, const TActorContext &ctx) +void TCreateTableUnit::Complete(TOperation::TPtr, const TActorContext &ctx) { - for (auto& ev : AddSenders) { - ctx.Send(DataShard.GetChangeSender(), ev.Release()); - } - - DataShard.MaybeActivateChangeSender(ctx); + for (auto& ev : AddSenders) { + ctx.Send(DataShard.GetChangeSender(), ev.Release()); + } + + DataShard.MaybeActivateChangeSender(ctx); } THolder<TExecutionUnit> CreateCreateTableUnit(TDataShard &dataShard, diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp index 2344c9aa726..d86568b4a15 100644 --- a/ydb/core/tx/datashard/datashard.cpp +++ b/ydb/core/tx/datashard/datashard.cpp @@ -111,8 +111,8 @@ TDataShard::TDataShard(const TActorId &tablet, TTabletStorageInfo *info) , State(TShardState::Uninitialized) , LastLocalTid(Schema::MinLocalTid) , LastSeqno(1) - , NextChangeRecordOrder(1) - , LastChangeRecordGroup(1) + , NextChangeRecordOrder(1) + , LastChangeRecordGroup(1) , TxReadSizeLimit(0) , StatisticsDisabled(0) , DisabledKeySampler(new NMiniKQL::TNoopKeySampler()) @@ -123,7 +123,7 @@ TDataShard::TDataShard(const TActorId &tablet, TTabletStorageInfo *info) , Pipeline(this) , SysLocks(this) , SnapshotManager(this) - , SchemaSnapshotManager(this) + , SchemaSnapshotManager(this) , DisableByKeyFilter(0, 0, 1) , MaxTxInFly(15000, 0, 100000) , MaxTxLagMilliseconds(5*60*1000, 0, 30*24*3600*1000ll) @@ -141,8 +141,8 @@ TDataShard::TDataShard(const TActorId &tablet, TTabletStorageInfo *info) , BackupReadAheadLo(0, 0, 64*1024*1024) , BackupReadAheadHi(0, 0, 128*1024*1024) , DataShardSysTables(InitDataShardSysTables(this)) - , ChangeSenderActivator(info->TabletID) - , ChangeExchangeSplitter(this) + , ChangeSenderActivator(info->TabletID) + , ChangeExchangeSplitter(this) { TabletCountersPtr.Reset(new TProtobufTabletCounters< ESimpleCounters_descriptor, @@ -166,7 +166,7 @@ NTabletPipe::TClientConfig TDataShard::GetPipeClientConfig() { } void TDataShard::OnDetach(const TActorContext &ctx) { - Cleanup(ctx); + Cleanup(ctx); LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "OnDetach: " << TabletID()); return Die(ctx); } @@ -258,24 +258,24 @@ void TDataShard::OnStopGuardComplete(const TActorContext &ctx) { void TDataShard::OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ev); LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "OnTabletDead: " << TabletID()); - Cleanup(ctx); + Cleanup(ctx); return Die(ctx); } -void TDataShard::Cleanup(const TActorContext& ctx) { +void TDataShard::Cleanup(const TActorContext& ctx) { //PipeClientCache->Detach(ctx); if (RegistrationSended) { - ctx.Send(MakeMediatorTimecastProxyID(), new TEvMediatorTimecast::TEvUnregisterTablet(TabletID())); + ctx.Send(MakeMediatorTimecastProxyID(), new TEvMediatorTimecast::TEvUnregisterTablet(TabletID())); + } + + if (Pipeline.HasRestore()) { + auto op = Pipeline.FindOp(Pipeline.CurrentSchemaTxId()); + if (op && op->IsWaitingForAsyncJob()) { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY(tx); + tx->KillAsyncJobActor(ctx); + } } - - if (Pipeline.HasRestore()) { - auto op = Pipeline.FindOp(Pipeline.CurrentSchemaTxId()); - if (op && op->IsWaitingForAsyncJob()) { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY(tx); - tx->KillAsyncJobActor(ctx); - } - } } void TDataShard::OnActivateExecutor(const TActorContext& ctx) { @@ -440,39 +440,39 @@ void TDataShard::FillExecutionStats(const TExecutionProfile& execProfile, TEvDat } ui64 TDataShard::AllocateChangeRecordOrder(NIceDb::TNiceDb& db) { - const ui64 result = NextChangeRecordOrder++; - PersistSys(db, Schema::Sys_NextChangeRecordOrder, NextChangeRecordOrder); - - return result; -} - + const ui64 result = NextChangeRecordOrder++; + PersistSys(db, Schema::Sys_NextChangeRecordOrder, NextChangeRecordOrder); + + return result; +} + ui64 TDataShard::AllocateChangeRecordGroup(NIceDb::TNiceDb& db) { - const ui64 now = TInstant::Now().GetValue(); - const ui64 result = now > LastChangeRecordGroup ? now : (LastChangeRecordGroup + 1); - - LastChangeRecordGroup = result; - PersistSys(db, Schema::Sys_LastChangeRecordGroup, LastChangeRecordGroup); - - return result; -} - + const ui64 now = TInstant::Now().GetValue(); + const ui64 result = now > LastChangeRecordGroup ? now : (LastChangeRecordGroup + 1); + + LastChangeRecordGroup = result; + PersistSys(db, Schema::Sys_LastChangeRecordGroup, LastChangeRecordGroup); + + return result; +} + void TDataShard::PersistChangeRecord(NIceDb::TNiceDb& db, const TChangeRecord& record) { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "PersistChangeRecord" << ": record: " << record << ", at tablet: " << TabletID()); - db.Table<Schema::ChangeRecords>().Key(record.GetOrder()).Update( - NIceDb::TUpdate<Schema::ChangeRecords::Group>(record.GetGroup()), - NIceDb::TUpdate<Schema::ChangeRecords::PlanStep>(record.GetStep()), - NIceDb::TUpdate<Schema::ChangeRecords::TxId>(record.GetTxId()), - NIceDb::TUpdate<Schema::ChangeRecords::PathOwnerId>(record.GetPathId().OwnerId), - NIceDb::TUpdate<Schema::ChangeRecords::LocalPathId>(record.GetPathId().LocalPathId), - NIceDb::TUpdate<Schema::ChangeRecords::BodySize>(record.GetBody().size())); - db.Table<Schema::ChangeRecordDetails>().Key(record.GetOrder()).Update( - NIceDb::TUpdate<Schema::ChangeRecordDetails::Kind>(record.GetKind()), - NIceDb::TUpdate<Schema::ChangeRecordDetails::Body>(record.GetBody())); -} - + db.Table<Schema::ChangeRecords>().Key(record.GetOrder()).Update( + NIceDb::TUpdate<Schema::ChangeRecords::Group>(record.GetGroup()), + NIceDb::TUpdate<Schema::ChangeRecords::PlanStep>(record.GetStep()), + NIceDb::TUpdate<Schema::ChangeRecords::TxId>(record.GetTxId()), + NIceDb::TUpdate<Schema::ChangeRecords::PathOwnerId>(record.GetPathId().OwnerId), + NIceDb::TUpdate<Schema::ChangeRecords::LocalPathId>(record.GetPathId().LocalPathId), + NIceDb::TUpdate<Schema::ChangeRecords::BodySize>(record.GetBody().size())); + db.Table<Schema::ChangeRecordDetails>().Key(record.GetOrder()).Update( + NIceDb::TUpdate<Schema::ChangeRecordDetails::Kind>(record.GetKind()), + NIceDb::TUpdate<Schema::ChangeRecordDetails::Body>(record.GetBody())); +} + void TDataShard::MoveChangeRecord(NIceDb::TNiceDb& db, ui64 order, const TPathId& pathId) { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "MoveChangeRecord" << ": order: " << order @@ -489,96 +489,96 @@ void TDataShard::RemoveChangeRecord(NIceDb::TNiceDb& db, ui64 order) { << ": order: " << order << ", at tablet: " << TabletID()); - auto it = ChangesQueue.find(order); - if (it != ChangesQueue.end()) { - Y_VERIFY(it->second <= ChangesQueueBytes); - ChangesQueueBytes -= it->second; - ChangesQueue.erase(it); - } - - db.Table<Schema::ChangeRecords>().Key(order).Delete(); - db.Table<Schema::ChangeRecordDetails>().Key(order).Delete(); -} - + auto it = ChangesQueue.find(order); + if (it != ChangesQueue.end()) { + Y_VERIFY(it->second <= ChangesQueueBytes); + ChangesQueueBytes -= it->second; + ChangesQueue.erase(it); + } + + db.Table<Schema::ChangeRecords>().Key(order).Delete(); + db.Table<Schema::ChangeRecordDetails>().Key(order).Delete(); +} + void TDataShard::EnqueueChangeRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records) { - if (!records) { - return; - } - - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "EnqueueChangeRecords" - << ": at tablet: " << TabletID() - << ", records: " << JoinSeq(", ", records)); - - for (const auto& record : records) { - if (ChangesQueue.emplace(record.Order, record.BodySize).second) { - Y_VERIFY(ChangesQueueBytes <= (Max<ui64>() - record.BodySize)); - ChangesQueueBytes += record.BodySize; - } - } - - Y_VERIFY(OutChangeSender); - Send(OutChangeSender, new TEvChangeExchange::TEvEnqueueRecords(std::move(records))); -} - + if (!records) { + return; + } + + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "EnqueueChangeRecords" + << ": at tablet: " << TabletID() + << ", records: " << JoinSeq(", ", records)); + + for (const auto& record : records) { + if (ChangesQueue.emplace(record.Order, record.BodySize).second) { + Y_VERIFY(ChangesQueueBytes <= (Max<ui64>() - record.BodySize)); + ChangesQueueBytes += record.BodySize; + } + } + + Y_VERIFY(OutChangeSender); + Send(OutChangeSender, new TEvChangeExchange::TEvEnqueueRecords(std::move(records))); +} + void TDataShard::CreateChangeSender(const TActorContext& ctx) { - Y_VERIFY(!OutChangeSender); + Y_VERIFY(!OutChangeSender); OutChangeSender = Register(NDataShard::CreateChangeSender(this)); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender created" - << ": at tablet: " << TabletID() - << ", actorId: " << OutChangeSender); -} - + + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender created" + << ": at tablet: " << TabletID() + << ", actorId: " << OutChangeSender); +} + void TDataShard::MaybeActivateChangeSender(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Trying to activate change sender" - << ": at tablet: " << TabletID()); - - if (ReceiveActivationsFrom) { - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" - << ": at tablet: " << TabletID() - << ", wait to activation from: " << JoinSeq(", ", ReceiveActivationsFrom)); - return; - } - - switch (State) { - case TShardState::WaitScheme: - case TShardState::SplitDstReceivingSnapshot: - case TShardState::Offline: - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" - << ": at tablet: " << TabletID() - << ", state: " << DatashardStateName(State)); - return; - - case TShardState::SplitSrcMakeSnapshot: - case TShardState::SplitSrcSendingSnapshot: - case TShardState::SplitSrcWaitForPartitioningChanged: - case TShardState::PreOffline: - if (!ChangesQueue) { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" - << ": at tablet: " << TabletID() - << ", state: " << DatashardStateName(State) - << ", queue size: " << ChangesQueue.size()); - return; - } - break; - } - - Y_VERIFY(OutChangeSender); - Send(OutChangeSender, new TEvChangeExchange::TEvActivateSender()); - - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender activated" - << ": at tablet: " << TabletID()); -} - + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Trying to activate change sender" + << ": at tablet: " << TabletID()); + + if (ReceiveActivationsFrom) { + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" + << ": at tablet: " << TabletID() + << ", wait to activation from: " << JoinSeq(", ", ReceiveActivationsFrom)); + return; + } + + switch (State) { + case TShardState::WaitScheme: + case TShardState::SplitDstReceivingSnapshot: + case TShardState::Offline: + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" + << ": at tablet: " << TabletID() + << ", state: " << DatashardStateName(State)); + return; + + case TShardState::SplitSrcMakeSnapshot: + case TShardState::SplitSrcSendingSnapshot: + case TShardState::SplitSrcWaitForPartitioningChanged: + case TShardState::PreOffline: + if (!ChangesQueue) { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot activate change sender" + << ": at tablet: " << TabletID() + << ", state: " << DatashardStateName(State) + << ", queue size: " << ChangesQueue.size()); + return; + } + break; + } + + Y_VERIFY(OutChangeSender); + Send(OutChangeSender, new TEvChangeExchange::TEvActivateSender()); + + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender activated" + << ": at tablet: " << TabletID()); +} + void TDataShard::KillChangeSender(const TActorContext& ctx) { - if (OutChangeSender) { - Send(std::exchange(OutChangeSender, TActorId()), new TEvents::TEvPoison()); - - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender killed" - << ": at tablet: " << TabletID()); - } -} - + if (OutChangeSender) { + Send(std::exchange(OutChangeSender, TActorId()), new TEvents::TEvPoison()); + + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "Change sender killed" + << ": at tablet: " << TabletID()); + } +} + bool TDataShard::LoadChangeRecords(NIceDb::TNiceDb& db, TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>& changeRecords) { using Schema = TDataShard::Schema; @@ -611,14 +611,14 @@ bool TDataShard::LoadChangeRecords(NIceDb::TNiceDb& db, TVector<TEvChangeExchang } void TDataShard::PersistSchemeTxResult(NIceDb::TNiceDb &db, const TSchemaOperation &op) { - db.Table<Schema::SchemaOperations>().Key(op.TxId).Update( - NIceDb::TUpdate<Schema::SchemaOperations::Success>(op.Success), - NIceDb::TUpdate<Schema::SchemaOperations::Error>(op.Error), - NIceDb::TUpdate<Schema::SchemaOperations::DataSize>(op.BytesProcessed), - NIceDb::TUpdate<Schema::SchemaOperations::Rows>(op.RowsProcessed) - ); -} - + db.Table<Schema::SchemaOperations>().Key(op.TxId).Update( + NIceDb::TUpdate<Schema::SchemaOperations::Success>(op.Success), + NIceDb::TUpdate<Schema::SchemaOperations::Error>(op.Error), + NIceDb::TUpdate<Schema::SchemaOperations::DataSize>(op.BytesProcessed), + NIceDb::TUpdate<Schema::SchemaOperations::Rows>(op.RowsProcessed) + ); +} + void TDataShard::NotifySchemeshard(const TActorContext& ctx, ui64 txId) { if (!txId) { for (const auto& op : TransQueue.GetSchemaOperations()) @@ -626,7 +626,7 @@ void TDataShard::NotifySchemeshard(const TActorContext& ctx, ui64 txId) { return; } - TSchemaOperation * op = TransQueue.FindSchemaTx(txId); + TSchemaOperation * op = TransQueue.FindSchemaTx(txId); if (!op || !op->Done) return; @@ -646,13 +646,13 @@ void TDataShard::NotifySchemeshard(const TActorContext& ctx, ui64 txId) { THolder(new TEvDataShard::TEvSchemaChanged(ctx.SelfID, TabletID(), State, op->TxId, op->PlanStep, Generation())); switch (op->Type) { - case TSchemaOperation::ETypeBackup: - case TSchemaOperation::ETypeRestore: { - auto* result = event->Record.MutableOpResult(); - result->SetSuccess(op->Success); - result->SetExplain(op->Error); - result->SetBytesProcessed(op->BytesProcessed); - result->SetRowsProcessed(op->RowsProcessed); + case TSchemaOperation::ETypeBackup: + case TSchemaOperation::ETypeRestore: { + auto* result = event->Record.MutableOpResult(); + result->SetSuccess(op->Success); + result->SetExplain(op->Error); + result->SetBytesProcessed(op->BytesProcessed); + result->SetRowsProcessed(op->RowsProcessed); break; } default: @@ -712,8 +712,8 @@ void TDataShard::PersistMoveUserTable(NIceDb::TNiceDb& db, ui64 prevTableId, ui6 } TUserTable::TPtr TDataShard::AlterTableSchemaVersion( - const TActorContext&, TTransactionContext& txc, - const TPathId& pathId, const ui64 tableSchemaVersion, bool persist) + const TActorContext&, TTransactionContext& txc, + const TPathId& pathId, const ui64 tableSchemaVersion, bool persist) { Y_VERIFY(GetPathOwnerId() == pathId.OwnerId); @@ -731,84 +731,84 @@ TUserTable::TPtr TDataShard::AlterTableSchemaVersion( << "old version " << oldTableInfo->GetTableSchemaVersion() << "new version " << newTableInfo->GetTableSchemaVersion()); - if (persist) { - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, tableId, *newTableInfo); - } + if (persist) { + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, tableId, *newTableInfo); + } return newTableInfo; } TUserTable::TPtr TDataShard::AlterTableAddIndex( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, const NKikimrSchemeOp::TIndexDescription& indexDesc) -{ - auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); - tableInfo->AddIndex(indexDesc); - - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, pathId.LocalPathId, *tableInfo); - - return tableInfo; -} - +{ + auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); + tableInfo->AddIndex(indexDesc); + + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, pathId.LocalPathId, *tableInfo); + + return tableInfo; +} + TUserTable::TPtr TDataShard::AlterTableDropIndex( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& indexPathId) -{ - auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); - tableInfo->DropIndex(indexPathId); - - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, pathId.LocalPathId, *tableInfo); - - return tableInfo; -} - + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& indexPathId) +{ + auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); + tableInfo->DropIndex(indexPathId); + + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, pathId.LocalPathId, *tableInfo); + + return tableInfo; +} + TUserTable::TPtr TDataShard::AlterTableAddCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, const NKikimrSchemeOp::TCdcStreamDescription& streamDesc) -{ - auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); - tableInfo->AddCdcStream(streamDesc); - - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, pathId.LocalPathId, *tableInfo); - - return tableInfo; -} - +{ + auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); + tableInfo->AddCdcStream(streamDesc); + + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, pathId.LocalPathId, *tableInfo); + + return tableInfo; +} + TUserTable::TPtr TDataShard::AlterTableDisableCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& streamPathId) -{ - auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); - tableInfo->DisableCdcStream(streamPathId); - - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, pathId.LocalPathId, *tableInfo); - - return tableInfo; -} - + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& streamPathId) +{ + auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); + tableInfo->DisableCdcStream(streamPathId); + + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, pathId.LocalPathId, *tableInfo); + + return tableInfo; +} + TUserTable::TPtr TDataShard::AlterTableDropCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& streamPathId) -{ - auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); - tableInfo->DropCdcStream(streamPathId); - - NIceDb::TNiceDb db(txc.DB); - PersistUserTable(db, pathId.LocalPathId, *tableInfo); - - return tableInfo; -} - + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& streamPathId) +{ + auto tableInfo = AlterTableSchemaVersion(ctx, txc, pathId, tableSchemaVersion, false); + tableInfo->DropCdcStream(streamPathId); + + NIceDb::TNiceDb db(txc.DB); + PersistUserTable(db, pathId.LocalPathId, *tableInfo); + + return tableInfo; +} + TUserTable::TPtr TDataShard::CreateUserTable(TTransactionContext& txc, const NKikimrSchemeOp::TTableDescription& tableScheme) { @@ -1517,7 +1517,7 @@ void TDataShard::Handle(TEvDataShard::TEvProposeTransaction::TPtr &ev, const TAc case NKikimrTxDataShard::TX_KIND_DATA: case NKikimrTxDataShard::TX_KIND_SCAN: case NKikimrTxDataShard::TX_KIND_SNAPSHOT: - case NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE: + case NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE: case NKikimrTxDataShard::TX_KIND_COMMIT_WRITES: ProposeTransaction(std::move(ev), ctx); return; @@ -1793,16 +1793,16 @@ void TDataShard::Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActo } } - if (ChangeSenderActivator.Has(ev->Get()->TabletId, ev->Get()->ClientId)) { - if (ev->Get()->Status != NKikimrProto::OK) { - if (!ev->Get()->Dead) { - ChangeSenderActivator.DoSend(ev->Get()->TabletId, ctx); - } else { - ChangeSenderActivator.AutoAck(ev->Get()->TabletId, ctx); - } - } - } - + if (ChangeSenderActivator.Has(ev->Get()->TabletId, ev->Get()->ClientId)) { + if (ev->Get()->Status != NKikimrProto::OK) { + if (!ev->Get()->Dead) { + ChangeSenderActivator.DoSend(ev->Get()->TabletId, ctx); + } else { + ChangeSenderActivator.AutoAck(ev->Get()->TabletId, ctx); + } + } + } + if (!PipeClientCache->OnConnect(ev)) { if (ev->Get()->Dead) { AckRSToDeletedTablet(ev->Get()->TabletId, ctx); @@ -1849,10 +1849,10 @@ void TDataShard::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActo SplitSrcSnapshotSender.DoSend(ev->Get()->TabletId, ctx); } - if (ChangeSenderActivator.Has(ev->Get()->TabletId, ev->Get()->ClientId)) { - ChangeSenderActivator.DoSend(ev->Get()->TabletId, ctx); - } - + if (ChangeSenderActivator.Has(ev->Get()->TabletId, ev->Get()->ClientId)) { + ChangeSenderActivator.DoSend(ev->Get()->TabletId, ctx); + } + LOG_DEBUG(ctx, NKikimrServices::TX_DATASHARD, "Client pipe to tablet %" PRIu64 " from %" PRIu64 " is reset", ev->Get()->TabletId, TabletID()); PipeClientCache->OnDisconnect(ev); RestartPipeRS(ev->Get()->TabletId, ctx); @@ -1959,12 +1959,12 @@ bool TDataShard::CheckTxNeedWait(const TEvDataShard::TEvProposeTransaction::TPtr } bool TDataShard::CheckChangesQueueOverflow() const { - const auto* appData = AppData(); - const auto sizeLimit = appData->DataShardConfig.GetChangesQueueItemsLimit(); - const auto bytesLimit = appData->DataShardConfig.GetChangesQueueBytesLimit(); - return ChangesQueue.size() >= sizeLimit || ChangesQueueBytes >= bytesLimit; -} - + const auto* appData = AppData(); + const auto sizeLimit = appData->DataShardConfig.GetChangesQueueItemsLimit(); + const auto bytesLimit = appData->DataShardConfig.GetChangesQueueBytesLimit(); + return ChangesQueue.size() >= sizeLimit || ChangesQueueBytes >= bytesLimit; +} + void TDataShard::Handle(TEvDataShard::TEvCancelTransactionProposal::TPtr &ev, const TActorContext &ctx) { ui64 txId = ev->Get()->Record.GetTxId(); LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Got TEvDataShard::TEvCancelTransactionProposal " << TabletID() @@ -2464,51 +2464,51 @@ void TDataShard::Handle(TEvDataShard::TEvMigrateSchemeShardRequest::TPtr& ev, } void TDataShard::Handle(TEvDataShard::TEvCancelBackup::TPtr& ev, const TActorContext& ctx) -{ - TOperation::TPtr op = Pipeline.FindOp(ev->Get()->Record.GetBackupTxId()); - if (op) { - ForwardEventToOperation(ev, op, ctx); - } -} - +{ + TOperation::TPtr op = Pipeline.FindOp(ev->Get()->Record.GetBackupTxId()); + if (op) { + ForwardEventToOperation(ev, op, ctx); + } +} + void TDataShard::Handle(TEvDataShard::TEvCancelRestore::TPtr& ev, const TActorContext& ctx) -{ - TOperation::TPtr op = Pipeline.FindOp(ev->Get()->Record.GetRestoreTxId()); - if (op) { - ForwardEventToOperation(ev, op, ctx); - } -} - +{ + TOperation::TPtr op = Pipeline.FindOp(ev->Get()->Record.GetRestoreTxId()); + if (op) { + ForwardEventToOperation(ev, op, ctx); + } +} + void TDataShard::Handle(TEvDataShard::TEvGetS3Upload::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxGetS3Upload(this, ev), ctx); -} - +{ + Execute(new TTxGetS3Upload(this, ev), ctx); +} + void TDataShard::Handle(TEvDataShard::TEvStoreS3UploadId::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxStoreS3UploadId(this, ev), ctx); -} - +{ + Execute(new TTxStoreS3UploadId(this, ev), ctx); +} + void TDataShard::Handle(TEvDataShard::TEvChangeS3UploadStatus::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxChangeS3UploadStatus(this, ev), ctx); -} - +{ + Execute(new TTxChangeS3UploadStatus(this, ev), ctx); +} + void TDataShard::Handle(TEvDataShard::TEvGetS3DownloadInfo::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxGetS3DownloadInfo(this, ev), ctx); -} - +{ + Execute(new TTxGetS3DownloadInfo(this, ev), ctx); +} + void TDataShard::Handle(TEvDataShard::TEvStoreS3DownloadInfo::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxStoreS3DownloadInfo(this, ev), ctx); -} - +{ + Execute(new TTxStoreS3DownloadInfo(this, ev), ctx); +} + void TDataShard::Handle(TEvDataShard::TEvUnsafeUploadRowsRequest::TPtr& ev, const TActorContext& ctx) -{ - Execute(new TTxUnsafeUploadRows(this, ev), ctx); -} - +{ + Execute(new TTxUnsafeUploadRows(this, ev), ctx); +} + void TDataShard::ScanComplete(NTable::EAbort, TAutoPtr<IDestructable> prod, ui64 cookie, @@ -2523,13 +2523,13 @@ void TDataShard::ScanComplete(NTable::EAbort, } else { LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "FullScan complete at " << TabletID()); - + auto op = Pipeline.FindOp(cookie); if (op) { LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Found op" - << ": cookie: " << cookie - << ", at: "<< TabletID()); - + << ": cookie: " << cookie + << ", at: "<< TabletID()); + if (op->IsWaitingForScan()) { op->SetScanResult(prod); Pipeline.AddCandidateOp(op); @@ -2553,29 +2553,29 @@ void TDataShard::ScanComplete(NTable::EAbort, } void TDataShard::Handle(TEvPrivate::TEvAsyncJobComplete::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "AsyncJob complete" - << " at " << TabletID()); - - auto op = Pipeline.FindOp(ev->Cookie); - if (op) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Found op" - << " at "<< TabletID() - << " cookie " << ev->Cookie); - - if (op->IsWaitingForAsyncJob()) { - op->SetAsyncJobResult(ev->Get()->Prod); - Pipeline.AddCandidateOp(op); - } - } else { - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "AsyncJob complete" - << " at " << TabletID() - << " for unknown tx " << ev->Cookie); - } - - // Continue current Tx - PlanQueue.Progress(ctx); -} - + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "AsyncJob complete" + << " at " << TabletID()); + + auto op = Pipeline.FindOp(ev->Cookie); + if (op) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Found op" + << " at "<< TabletID() + << " cookie " << ev->Cookie); + + if (op->IsWaitingForAsyncJob()) { + op->SetAsyncJobResult(ev->Get()->Prod); + Pipeline.AddCandidateOp(op); + } + } else { + LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "AsyncJob complete" + << " at " << TabletID() + << " for unknown tx " << ev->Cookie); + } + + // Continue current Tx + PlanQueue.Progress(ctx); +} + bool TDataShard::ReassignChannelsEnabled() const { return true; } diff --git a/ydb/core/tx/datashard/datashard.h b/ydb/core/tx/datashard/datashard.h index 3af6723b225..5a93c035026 100644 --- a/ydb/core/tx/datashard/datashard.h +++ b/ydb/core/tx/datashard/datashard.h @@ -1,6 +1,6 @@ #pragma once - -#include "datashard_s3_upload.h" + +#include "datashard_s3_upload.h" #include <ydb/core/tx/tx.h> #include <ydb/core/tx/message_seqno.h> @@ -112,8 +112,8 @@ namespace NDataShard { ResultSent = 1ULL << 41, // Operation is confirmed to be stored in shard's local database Stored = 1ULL << 42, - // Operation is waiting for async job to finish - WaitingForAsyncJob = 1ULL << 43, + // Operation is waiting for async job to finish + WaitingForAsyncJob = 1ULL << 43, // Operation must complete before results sending WaitCompletion = 1ULL << 44, @@ -200,7 +200,7 @@ struct TEvDataShard { EvCancelBackup, EvMigrateSchemeShardRequest, EvMigrateSchemeShardResponse, - EvCancelRestore, + EvCancelRestore, EvInitDataShardResult = EvProposeTransaction + 5 * 512, EvGetShardStateResult, @@ -258,29 +258,29 @@ struct TEvDataShard { EvDiscardVolatileSnapshotRequest, EvDiscardVolatileSnapshotResponse, - EvGetS3Upload, - EvStoreS3UploadId, - EvS3Upload, - - EvEraseRowsRequest, - EvEraseRowsResponse, - EvConditionalEraseRowsRequest, - EvConditionalEraseRowsResponse, - + EvGetS3Upload, + EvStoreS3UploadId, + EvS3Upload, + + EvEraseRowsRequest, + EvEraseRowsResponse, + EvConditionalEraseRowsRequest, + EvConditionalEraseRowsResponse, + EvBuildIndexCreateRequest, EvBuildIndexProgressResponse, - EvGetS3DownloadInfo, - EvStoreS3DownloadInfo, - EvS3DownloadInfo, - - EvUnsafeUploadRowsRequest, - EvUnsafeUploadRowsResponse, - + EvGetS3DownloadInfo, + EvStoreS3DownloadInfo, + EvS3DownloadInfo, + + EvUnsafeUploadRowsRequest, + EvUnsafeUploadRowsResponse, + EvKqpScan, - EvChangeS3UploadStatus, - + EvChangeS3UploadStatus, + EvGetRemovedRowVersions, /* for tests */ EvGetRemovedRowVersionsResult, /* for tests */ @@ -775,19 +775,19 @@ struct TEvDataShard { } }; - struct TEvCancelRestore - : public TEventPB<TEvCancelRestore, - NKikimrTxDataShard::TEvCancelRestore, - TEvDataShard::EvCancelRestore> - { - TEvCancelRestore() = default; - - explicit TEvCancelRestore(ui64 txid, ui64 tableId) { - Record.SetRestoreTxId(txid); - Record.SetTableId(tableId); - } - }; - + struct TEvCancelRestore + : public TEventPB<TEvCancelRestore, + NKikimrTxDataShard::TEvCancelRestore, + TEvDataShard::EvCancelRestore> + { + TEvCancelRestore() = default; + + explicit TEvCancelRestore(ui64 txid, ui64 tableId) { + Record.SetRestoreTxId(txid); + Record.SetTableId(tableId); + } + }; + struct TEvGetTableStats : public TEventPB<TEvGetTableStats, NKikimrTxDataShard::TEvGetTableStats, TEvDataShard::EvGetTableStats> { @@ -857,24 +857,24 @@ struct TEvDataShard { } }; - struct TEvUnsafeUploadRowsRequest : public TEventPBWithArena<TEvUnsafeUploadRowsRequest, - NKikimrTxDataShard::TEvUploadRowsRequest, - TEvDataShard::EvUnsafeUploadRowsRequest, - 16200, 32500> { - TEvUnsafeUploadRowsRequest() = default; - }; + struct TEvUnsafeUploadRowsRequest : public TEventPBWithArena<TEvUnsafeUploadRowsRequest, + NKikimrTxDataShard::TEvUploadRowsRequest, + TEvDataShard::EvUnsafeUploadRowsRequest, + 16200, 32500> { + TEvUnsafeUploadRowsRequest() = default; + }; - struct TEvUnsafeUploadRowsResponse : public TEventPB<TEvUnsafeUploadRowsResponse, - NKikimrTxDataShard::TEvUploadRowsResponse, - TEvDataShard::EvUnsafeUploadRowsResponse> { - TEvUnsafeUploadRowsResponse() = default; + struct TEvUnsafeUploadRowsResponse : public TEventPB<TEvUnsafeUploadRowsResponse, + NKikimrTxDataShard::TEvUploadRowsResponse, + TEvDataShard::EvUnsafeUploadRowsResponse> { + TEvUnsafeUploadRowsResponse() = default; + + explicit TEvUnsafeUploadRowsResponse(ui64 tabletId, ui32 status = NKikimrTxDataShard::TError::OK) { + Record.SetTabletID(tabletId); + Record.SetStatus(status); + } + }; - explicit TEvUnsafeUploadRowsResponse(ui64 tabletId, ui32 status = NKikimrTxDataShard::TError::OK) { - Record.SetTabletID(tabletId); - Record.SetStatus(status); - } - }; - // In most cases this event is local, thus users must // use Keys, Ranges and Program struct members instead of corresponding // protobuf members. In case of remote event these struct members will @@ -1148,190 +1148,190 @@ struct TEvDataShard { { using EStatus = NKikimrTxDataShard::TEvMigrateSchemeShardResponse::EStatus; }; - - struct TEvGetS3Upload - : public TEventLocal<TEvGetS3Upload, TEvDataShard::EvGetS3Upload> - { + + struct TEvGetS3Upload + : public TEventLocal<TEvGetS3Upload, TEvDataShard::EvGetS3Upload> + { TActorId ReplyTo; - ui64 TxId; - - explicit TEvGetS3Upload(const TActorId& replyTo, ui64 txId) - : ReplyTo(replyTo) - , TxId(txId) - { - } - }; - - struct TEvStoreS3UploadId - : public TEventLocal<TEvStoreS3UploadId, TEvDataShard::EvStoreS3UploadId> - { + ui64 TxId; + + explicit TEvGetS3Upload(const TActorId& replyTo, ui64 txId) + : ReplyTo(replyTo) + , TxId(txId) + { + } + }; + + struct TEvStoreS3UploadId + : public TEventLocal<TEvStoreS3UploadId, TEvDataShard::EvStoreS3UploadId> + { TActorId ReplyTo; - ui64 TxId; - TString UploadId; - + ui64 TxId; + TString UploadId; + explicit TEvStoreS3UploadId(const TActorId& replyTo, ui64 txId, const TString& uploadId) - : ReplyTo(replyTo) - , TxId(txId) - , UploadId(uploadId) - { - } - }; - - struct TEvChangeS3UploadStatus - : public TEventLocal<TEvChangeS3UploadStatus, TEvDataShard::EvChangeS3UploadStatus> - { + : ReplyTo(replyTo) + , TxId(txId) + , UploadId(uploadId) + { + } + }; + + struct TEvChangeS3UploadStatus + : public TEventLocal<TEvChangeS3UploadStatus, TEvDataShard::EvChangeS3UploadStatus> + { using EStatus = NDataShard::TS3Upload::EStatus; - - TActorId ReplyTo; - ui64 TxId; - EStatus Status; - TMaybe<TString> Error; - TVector<TString> Parts; - - explicit TEvChangeS3UploadStatus(const TActorId& replyTo, ui64 txId, EStatus status, TVector<TString>&& parts) - : ReplyTo(replyTo) - , TxId(txId) - , Status(status) - , Parts(std::move(parts)) - { - } - - explicit TEvChangeS3UploadStatus(const TActorId& replyTo, ui64 txId, EStatus status, const TString& error) - : ReplyTo(replyTo) - , TxId(txId) - , Status(status) - , Error(error) - { - } - }; - - struct TEvS3Upload - : public TEventLocal<TEvS3Upload, TEvDataShard::EvS3Upload> - { + + TActorId ReplyTo; + ui64 TxId; + EStatus Status; + TMaybe<TString> Error; + TVector<TString> Parts; + + explicit TEvChangeS3UploadStatus(const TActorId& replyTo, ui64 txId, EStatus status, TVector<TString>&& parts) + : ReplyTo(replyTo) + , TxId(txId) + , Status(status) + , Parts(std::move(parts)) + { + } + + explicit TEvChangeS3UploadStatus(const TActorId& replyTo, ui64 txId, EStatus status, const TString& error) + : ReplyTo(replyTo) + , TxId(txId) + , Status(status) + , Error(error) + { + } + }; + + struct TEvS3Upload + : public TEventLocal<TEvS3Upload, TEvDataShard::EvS3Upload> + { using TS3Upload = NDataShard::TS3Upload; - - TMaybe<TS3Upload> Upload; - - TEvS3Upload() = default; - - explicit TEvS3Upload(const TS3Upload& upload) - : Upload(upload) - { - } - }; - - struct TEvGetS3DownloadInfo - : public TEventLocal<TEvGetS3DownloadInfo, TEvDataShard::EvGetS3DownloadInfo> - { - TActorId ReplyTo; - ui64 TxId; - - explicit TEvGetS3DownloadInfo(const TActorId& replyTo, ui64 txId) - : ReplyTo(replyTo) - , TxId(txId) - { - } - }; - - struct TEvStoreS3DownloadInfo - : public TEventLocal<TEvStoreS3DownloadInfo, TEvDataShard::EvStoreS3DownloadInfo> - { - TActorId ReplyTo; - ui64 TxId; - TString DataETag; - ui64 ProcessedBytes; - ui64 WrittenBytes; - ui64 WrittenRows; - - TEvStoreS3DownloadInfo() = default; - - explicit TEvStoreS3DownloadInfo( - const TActorId& replyTo, - ui64 txId, - const TString& dataETag, - ui64 processedBytes, - ui64 writtenBytes, - ui64 writtenRows) - : ReplyTo(replyTo) - , TxId(txId) - , DataETag(dataETag) - , ProcessedBytes(processedBytes) - , WrittenBytes(writtenBytes) - , WrittenRows(writtenRows) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " {" - << " TxId: " << TxId - << " DataETag: " << DataETag - << " ProcessedBytes: " << ProcessedBytes - << " WrittenBytes: " << WrittenBytes - << " WrittenRows: " << WrittenRows - << " }"; - } - }; - - struct TEvS3DownloadInfo - : public TEventLocal<TEvS3DownloadInfo, TEvDataShard::EvS3DownloadInfo> - { - struct TInfo { - TMaybe<TString> DataETag; - ui64 ProcessedBytes = 0; - ui64 WrittenBytes = 0; - ui64 WrittenRows = 0; - - TString ToString() const { - return TStringBuilder() << "{" - << " DataETag: " << DataETag - << " ProcessedBytes: " << ProcessedBytes - << " WrittenBytes: " << WrittenBytes - << " WrittenRows: " << WrittenRows - << " }"; - } - }; - - TInfo Info; - - TEvS3DownloadInfo() = default; - - explicit TEvS3DownloadInfo(const TInfo& info) - : Info(info) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " " << Info.ToString(); - } - }; - - struct TEvEraseRowsRequest - : public TEventPB<TEvEraseRowsRequest, - NKikimrTxDataShard::TEvEraseRowsRequest, - TEvDataShard::EvEraseRowsRequest> - { - }; - - struct TEvEraseRowsResponse - : public TEventPB<TEvEraseRowsResponse, - NKikimrTxDataShard::TEvEraseRowsResponse, - TEvDataShard::EvEraseRowsResponse> - { - }; - - struct TEvConditionalEraseRowsRequest - : public TEventPB<TEvConditionalEraseRowsRequest, - NKikimrTxDataShard::TEvConditionalEraseRowsRequest, - TEvDataShard::EvConditionalEraseRowsRequest> - { - }; - - struct TEvConditionalEraseRowsResponse - : public TEventPB<TEvConditionalEraseRowsResponse, - NKikimrTxDataShard::TEvConditionalEraseRowsResponse, - TEvDataShard::EvConditionalEraseRowsResponse> - { - }; + + TMaybe<TS3Upload> Upload; + + TEvS3Upload() = default; + + explicit TEvS3Upload(const TS3Upload& upload) + : Upload(upload) + { + } + }; + + struct TEvGetS3DownloadInfo + : public TEventLocal<TEvGetS3DownloadInfo, TEvDataShard::EvGetS3DownloadInfo> + { + TActorId ReplyTo; + ui64 TxId; + + explicit TEvGetS3DownloadInfo(const TActorId& replyTo, ui64 txId) + : ReplyTo(replyTo) + , TxId(txId) + { + } + }; + + struct TEvStoreS3DownloadInfo + : public TEventLocal<TEvStoreS3DownloadInfo, TEvDataShard::EvStoreS3DownloadInfo> + { + TActorId ReplyTo; + ui64 TxId; + TString DataETag; + ui64 ProcessedBytes; + ui64 WrittenBytes; + ui64 WrittenRows; + + TEvStoreS3DownloadInfo() = default; + + explicit TEvStoreS3DownloadInfo( + const TActorId& replyTo, + ui64 txId, + const TString& dataETag, + ui64 processedBytes, + ui64 writtenBytes, + ui64 writtenRows) + : ReplyTo(replyTo) + , TxId(txId) + , DataETag(dataETag) + , ProcessedBytes(processedBytes) + , WrittenBytes(writtenBytes) + , WrittenRows(writtenRows) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " TxId: " << TxId + << " DataETag: " << DataETag + << " ProcessedBytes: " << ProcessedBytes + << " WrittenBytes: " << WrittenBytes + << " WrittenRows: " << WrittenRows + << " }"; + } + }; + + struct TEvS3DownloadInfo + : public TEventLocal<TEvS3DownloadInfo, TEvDataShard::EvS3DownloadInfo> + { + struct TInfo { + TMaybe<TString> DataETag; + ui64 ProcessedBytes = 0; + ui64 WrittenBytes = 0; + ui64 WrittenRows = 0; + + TString ToString() const { + return TStringBuilder() << "{" + << " DataETag: " << DataETag + << " ProcessedBytes: " << ProcessedBytes + << " WrittenBytes: " << WrittenBytes + << " WrittenRows: " << WrittenRows + << " }"; + } + }; + + TInfo Info; + + TEvS3DownloadInfo() = default; + + explicit TEvS3DownloadInfo(const TInfo& info) + : Info(info) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " " << Info.ToString(); + } + }; + + struct TEvEraseRowsRequest + : public TEventPB<TEvEraseRowsRequest, + NKikimrTxDataShard::TEvEraseRowsRequest, + TEvDataShard::EvEraseRowsRequest> + { + }; + + struct TEvEraseRowsResponse + : public TEventPB<TEvEraseRowsResponse, + NKikimrTxDataShard::TEvEraseRowsResponse, + TEvDataShard::EvEraseRowsResponse> + { + }; + + struct TEvConditionalEraseRowsRequest + : public TEventPB<TEvConditionalEraseRowsRequest, + NKikimrTxDataShard::TEvConditionalEraseRowsRequest, + TEvDataShard::EvConditionalEraseRowsRequest> + { + }; + + struct TEvConditionalEraseRowsResponse + : public TEventPB<TEvConditionalEraseRowsResponse, + NKikimrTxDataShard::TEvConditionalEraseRowsResponse, + TEvDataShard::EvConditionalEraseRowsResponse> + { + }; struct TEvBuildIndexCreateRequest : public TEventPB<TEvBuildIndexCreateRequest, diff --git a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp index 06688af51dc..026b14e461f 100644 --- a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp @@ -1,612 +1,612 @@ -#include "datashard_distributed_erase.h" -#include "datashard_impl.h" -#include "erase_rows_condition.h" - +#include "datashard_distributed_erase.h" +#include "datashard_impl.h" +#include "erase_rows_condition.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/tablet_flat/flat_row_state.h> - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/vector.h> -#include <util/stream/output.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/vector.h> +#include <util/stream/output.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NActors; -using namespace NTable; - -using TLimits = NKikimrTxDataShard::TEvConditionalEraseRowsRequest::TLimits; - -class IEraserOps { -protected: - struct TKey { - TTag Tag = Max<TTag>(); - TPos Pos = Max<TPos>(); - }; - + +using namespace NActors; +using namespace NTable; + +using TLimits = NKikimrTxDataShard::TEvConditionalEraseRowsRequest::TLimits; + +class IEraserOps { +protected: + struct TKey { + TTag Tag = Max<TTag>(); + TPos Pos = Max<TPos>(); + }; + virtual TVector<TKey> MakeKeyOrder(TIntrusiveConstPtr<IScan::TScheme> scheme) const = 0; - virtual TActorId CreateEraser() = 0; - virtual void CloseEraser() = 0; -}; - -class TCondEraseScan: public IActor, public IScan, public IEraserOps { - struct TDataShardId { + virtual TActorId CreateEraser() = 0; + virtual void CloseEraser() = 0; +}; + +class TCondEraseScan: public IActor, public IScan, public IEraserOps { + struct TDataShardId { TActorId ActorId; - ui64 TabletId; - }; - - class TSerializedKeys { - public: - explicit TSerializedKeys( - ui32 bytesLimit = 500 * 1024, - ui32 minCount = 1000, - ui32 maxCount = 50000) - : BytesLimit(bytesLimit) - , MinCount(minCount) - , MaxCount(maxCount) - , Size(0) - { - } - - void Add(TString key) { - Size += key.size(); - Keys.emplace_back(std::move(key)); - } - - void Clear() { - Keys.clear(); - Size = 0; - } - - TVector<TString>& GetKeys() { - return Keys; - } - - ui32 Count() const { - return Keys.size(); - } - - ui32 Bytes() const { - return Size; - } - - bool CheckLimits() const { - if (Size < BytesLimit) { - if (Count() < MaxCount) { - return true; - } - } else { - if (Count() < MinCount) { - return true; - } - } - - return false; - } - - explicit operator bool() const { - return Count(); - } - - private: - const ui32 BytesLimit; - const ui32 MinCount; - const ui32 MaxCount; - - TVector<TString> Keys; - ui32 Size; - }; - - struct TStats { - TStats() - : RowsProcessed(0) - , RowsErased(0) - { - auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "erase_rows"); - - MonProcessed = counters->GetCounter("Processed", true); - MonErased = counters->GetCounter("Erased", true); - } - - void IncProcessed() { - ++RowsProcessed; - *MonProcessed += 1; - } - - void IncErased() { - ++RowsErased; - *MonErased += 1; - } - - void ToProto(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::TStats& stats) const { - stats.SetRowsProcessed(RowsProcessed); - stats.SetRowsErased(RowsErased); - } - - private: - ui64 RowsProcessed; - ui64 RowsErased; - NMonitoring::TDynamicCounters::TCounterPtr MonProcessed; - NMonitoring::TDynamicCounters::TCounterPtr MonErased; - }; - - static TVector<TCell> MakeKeyCells(const TVector<TKey>& keyOrder, const TRow& row) { - TVector<TCell> keyCells; - - for (const auto& key : keyOrder) { - Y_VERIFY(key.Pos != Max<TPos>()); - Y_VERIFY(key.Pos < row.Size()); - keyCells.push_back(row.Get(key.Pos)); - } - - return keyCells; - } - - static THolder<TEvDataShard::TEvEraseRowsRequest> MakeEraseRowsRequest(const TTableId& tableId, - IEraseRowsCondition* condition, const TVector<TKey>& keyOrder, TVector<TString>& keys) - { - auto request = MakeHolder<TEvDataShard::TEvEraseRowsRequest>(); - - request->Record.SetTableId(tableId.PathId.LocalPathId); - request->Record.SetSchemaVersion(tableId.SchemaVersion); - - for (const auto& key : keyOrder) { - Y_VERIFY(key.Tag != Max<TTag>()); - request->Record.AddKeyColumnIds(key.Tag); - } - - for (TString& key : keys) { - request->Record.AddKeyColumns(std::move(key)); - } - - condition->AddToRequest(request->Record); - - return request; - } - - void SendEraseRowsRequest() { - Send(CreateEraser(), MakeEraseRowsRequest(TableId, Condition.Get(), KeyOrder, SerializedKeys.GetKeys())); - SerializedKeys.Clear(); - } - - void Reply(bool aborted = false) { - auto response = MakeHolder<TEvDataShard::TEvConditionalEraseRowsResponse>(); - response->Record.SetTabletID(DataShard.TabletId); - - if (aborted) { - response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ABORTED); - } else if (!Success) { - response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ERASE_ERROR); - } else if (!NoMoreData) { - response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::PARTIAL); - } else { - response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OK); - } - - response->Record.SetErrorDescription(Error); - Stats.ToProto(*response->Record.MutableStats()); - - Send(ReplyTo, std::move(response)); - } - - void Handle(TEvDataShard::TEvEraseRowsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - Success = (record.GetStatus() == NKikimrTxDataShard::TEvEraseRowsResponse::OK); - Error = record.GetErrorDescription(); - - CloseEraser(); - - if (NoMoreData || !Success) { - Driver->Touch(EScan::Final); - } else { - Reply(); - Driver->Touch(EScan::Feed); - } - } - - void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev) { - ReplyTo = ev->Sender; - Reply(); - } - -public: + ui64 TabletId; + }; + + class TSerializedKeys { + public: + explicit TSerializedKeys( + ui32 bytesLimit = 500 * 1024, + ui32 minCount = 1000, + ui32 maxCount = 50000) + : BytesLimit(bytesLimit) + , MinCount(minCount) + , MaxCount(maxCount) + , Size(0) + { + } + + void Add(TString key) { + Size += key.size(); + Keys.emplace_back(std::move(key)); + } + + void Clear() { + Keys.clear(); + Size = 0; + } + + TVector<TString>& GetKeys() { + return Keys; + } + + ui32 Count() const { + return Keys.size(); + } + + ui32 Bytes() const { + return Size; + } + + bool CheckLimits() const { + if (Size < BytesLimit) { + if (Count() < MaxCount) { + return true; + } + } else { + if (Count() < MinCount) { + return true; + } + } + + return false; + } + + explicit operator bool() const { + return Count(); + } + + private: + const ui32 BytesLimit; + const ui32 MinCount; + const ui32 MaxCount; + + TVector<TString> Keys; + ui32 Size; + }; + + struct TStats { + TStats() + : RowsProcessed(0) + , RowsErased(0) + { + auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "erase_rows"); + + MonProcessed = counters->GetCounter("Processed", true); + MonErased = counters->GetCounter("Erased", true); + } + + void IncProcessed() { + ++RowsProcessed; + *MonProcessed += 1; + } + + void IncErased() { + ++RowsErased; + *MonErased += 1; + } + + void ToProto(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::TStats& stats) const { + stats.SetRowsProcessed(RowsProcessed); + stats.SetRowsErased(RowsErased); + } + + private: + ui64 RowsProcessed; + ui64 RowsErased; + NMonitoring::TDynamicCounters::TCounterPtr MonProcessed; + NMonitoring::TDynamicCounters::TCounterPtr MonErased; + }; + + static TVector<TCell> MakeKeyCells(const TVector<TKey>& keyOrder, const TRow& row) { + TVector<TCell> keyCells; + + for (const auto& key : keyOrder) { + Y_VERIFY(key.Pos != Max<TPos>()); + Y_VERIFY(key.Pos < row.Size()); + keyCells.push_back(row.Get(key.Pos)); + } + + return keyCells; + } + + static THolder<TEvDataShard::TEvEraseRowsRequest> MakeEraseRowsRequest(const TTableId& tableId, + IEraseRowsCondition* condition, const TVector<TKey>& keyOrder, TVector<TString>& keys) + { + auto request = MakeHolder<TEvDataShard::TEvEraseRowsRequest>(); + + request->Record.SetTableId(tableId.PathId.LocalPathId); + request->Record.SetSchemaVersion(tableId.SchemaVersion); + + for (const auto& key : keyOrder) { + Y_VERIFY(key.Tag != Max<TTag>()); + request->Record.AddKeyColumnIds(key.Tag); + } + + for (TString& key : keys) { + request->Record.AddKeyColumns(std::move(key)); + } + + condition->AddToRequest(request->Record); + + return request; + } + + void SendEraseRowsRequest() { + Send(CreateEraser(), MakeEraseRowsRequest(TableId, Condition.Get(), KeyOrder, SerializedKeys.GetKeys())); + SerializedKeys.Clear(); + } + + void Reply(bool aborted = false) { + auto response = MakeHolder<TEvDataShard::TEvConditionalEraseRowsResponse>(); + response->Record.SetTabletID(DataShard.TabletId); + + if (aborted) { + response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ABORTED); + } else if (!Success) { + response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ERASE_ERROR); + } else if (!NoMoreData) { + response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::PARTIAL); + } else { + response->Record.SetStatus(NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OK); + } + + response->Record.SetErrorDescription(Error); + Stats.ToProto(*response->Record.MutableStats()); + + Send(ReplyTo, std::move(response)); + } + + void Handle(TEvDataShard::TEvEraseRowsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + Success = (record.GetStatus() == NKikimrTxDataShard::TEvEraseRowsResponse::OK); + Error = record.GetErrorDescription(); + + CloseEraser(); + + if (NoMoreData || !Success) { + Driver->Touch(EScan::Final); + } else { + Reply(); + Driver->Touch(EScan::Feed); + } + } + + void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev) { + ReplyTo = ev->Sender; + Reply(); + } + +public: explicit TCondEraseScan(TDataShard* ds, const TActorId& replyTo, const TTableId& tableId, ui64 txId, THolder<IEraseRowsCondition> condition, const TLimits& limits) - : IActor(static_cast<TReceiveFunc>(&TCondEraseScan::StateWork), NKikimrServices::TActivity::CONDITIONAL_ERASE_ROWS_SCAN_ACTOR) - , TableId(tableId) - , DataShard{ds->SelfId(), ds->TabletID()} - , ReplyTo(replyTo) - , TxId(txId) - , Condition(std::move(condition)) - , Driver(nullptr) - , SerializedKeys(limits.GetBatchMaxBytes(), limits.GetBatchMinKeys(), limits.GetBatchMaxKeys()) - , NoMoreData(false) - , Success(true) - { - } - - void Describe(IOutputStream& o) const noexcept override { - o << "CondEraseScan {" - << " TableId: " << TableId - << " TxId: " << TxId - << " }"; - } - + : IActor(static_cast<TReceiveFunc>(&TCondEraseScan::StateWork), NKikimrServices::TActivity::CONDITIONAL_ERASE_ROWS_SCAN_ACTOR) + , TableId(tableId) + , DataShard{ds->SelfId(), ds->TabletID()} + , ReplyTo(replyTo) + , TxId(txId) + , Condition(std::move(condition)) + , Driver(nullptr) + , SerializedKeys(limits.GetBatchMaxBytes(), limits.GetBatchMinKeys(), limits.GetBatchMaxKeys()) + , NoMoreData(false) + , Success(true) + { + } + + void Describe(IOutputStream& o) const noexcept override { + o << "CondEraseScan {" + << " TableId: " << TableId + << " TxId: " << TxId + << " }"; + } + IScan::THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept override { TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); - - Driver = driver; - Scheme = std::move(scheme); - KeyOrder = MakeKeyOrder(Scheme); // fill tags - - // fill scan tags & positions in KeyOrder - ScanTags = Condition->Tags(); - Y_VERIFY(ScanTags.size() == 1, "Multi-column conditions are not supported"); - - THashMap<TTag, TPos> tagToPos; - - for (TPos pos = 0; pos < ScanTags.size(); ++pos) { - Y_VERIFY(tagToPos.emplace(ScanTags.at(pos), pos).second); - } - - for (auto& key : KeyOrder) { - auto it = tagToPos.find(key.Tag); - if (it == tagToPos.end()) { - it = tagToPos.emplace(key.Tag, ScanTags.size()).first; - ScanTags.push_back(key.Tag); - } - - key.Pos = it->second; - } - - Condition->Prepare(Scheme, 0); - - return {EScan::Feed, {}}; - } - - void Registered(TActorSystem* sys, const TActorId&) override { + + Driver = driver; + Scheme = std::move(scheme); + KeyOrder = MakeKeyOrder(Scheme); // fill tags + + // fill scan tags & positions in KeyOrder + ScanTags = Condition->Tags(); + Y_VERIFY(ScanTags.size() == 1, "Multi-column conditions are not supported"); + + THashMap<TTag, TPos> tagToPos; + + for (TPos pos = 0; pos < ScanTags.size(); ++pos) { + Y_VERIFY(tagToPos.emplace(ScanTags.at(pos), pos).second); + } + + for (auto& key : KeyOrder) { + auto it = tagToPos.find(key.Tag); + if (it == tagToPos.end()) { + it = tagToPos.emplace(key.Tag, ScanTags.size()).first; + ScanTags.push_back(key.Tag); + } + + key.Pos = it->second; + } + + Condition->Prepare(Scheme, 0); + + return {EScan::Feed, {}}; + } + + void Registered(TActorSystem* sys, const TActorId&) override { sys->Send(DataShard.ActorId, new TDataShard::TEvPrivate::TEvConditionalEraseRowsRegistered(TxId, SelfId())); - } - - EScan Seek(TLead& lead, ui64) noexcept override { - lead.To(ScanTags, {}, ESeek::Lower); - return EScan::Feed; - } - - EScan Feed(TArrayRef<const TCell>, const TRow& row) noexcept override { - Stats.IncProcessed(); - if (!Condition->Check(row)) { - return EScan::Feed; - } - - Stats.IncErased(); - SerializedKeys.Add(TSerializedCellVec::Serialize(MakeKeyCells(KeyOrder, row))); - if (SerializedKeys.CheckLimits()) { - return EScan::Feed; - } - - SendEraseRowsRequest(); - return EScan::Sleep; - } - - EScan Exhausted() noexcept override { - NoMoreData = true; - - if (!SerializedKeys) { - return EScan::Final; - } - - SendEraseRowsRequest(); - return EScan::Sleep; - } - + } + + EScan Seek(TLead& lead, ui64) noexcept override { + lead.To(ScanTags, {}, ESeek::Lower); + return EScan::Feed; + } + + EScan Feed(TArrayRef<const TCell>, const TRow& row) noexcept override { + Stats.IncProcessed(); + if (!Condition->Check(row)) { + return EScan::Feed; + } + + Stats.IncErased(); + SerializedKeys.Add(TSerializedCellVec::Serialize(MakeKeyCells(KeyOrder, row))); + if (SerializedKeys.CheckLimits()) { + return EScan::Feed; + } + + SendEraseRowsRequest(); + return EScan::Sleep; + } + + EScan Exhausted() noexcept override { + NoMoreData = true; + + if (!SerializedKeys) { + return EScan::Final; + } + + SendEraseRowsRequest(); + return EScan::Sleep; + } + TAutoPtr<IDestructable> Finish(EAbort abort) noexcept override { - Reply(abort != EAbort::None); - PassAway(); - - return nullptr; - } - - void PassAway() override { - CloseEraser(); - IActor::PassAway(); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvDataShard::TEvEraseRowsResponse, Handle); - hFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle); - } - } - -protected: + Reply(abort != EAbort::None); + PassAway(); + + return nullptr; + } + + void PassAway() override { + CloseEraser(); + IActor::PassAway(); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvDataShard::TEvEraseRowsResponse, Handle); + hFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle); + } + } + +protected: TVector<TKey> MakeKeyOrder(TIntrusiveConstPtr<IScan::TScheme> scheme) const override { - TVector<TKey> keyOrder; - - for (const auto& col : scheme->Cols) { - if (!col.IsKey()) { - continue; - } - - if (keyOrder.size() < (col.Key + 1)) { - keyOrder.resize(col.Key + 1); - } - - keyOrder[col.Key].Tag = col.Tag; - } - - return keyOrder; - } - - TActorId CreateEraser() override { - return DataShard.ActorId; - } - - void CloseEraser() override { - // nop - } - -protected: - const TTableId TableId; - -private: - const TDataShardId DataShard; + TVector<TKey> keyOrder; + + for (const auto& col : scheme->Cols) { + if (!col.IsKey()) { + continue; + } + + if (keyOrder.size() < (col.Key + 1)) { + keyOrder.resize(col.Key + 1); + } + + keyOrder[col.Key].Tag = col.Tag; + } + + return keyOrder; + } + + TActorId CreateEraser() override { + return DataShard.ActorId; + } + + void CloseEraser() override { + // nop + } + +protected: + const TTableId TableId; + +private: + const TDataShardId DataShard; TActorId ReplyTo; - const ui64 TxId; - THolder<IEraseRowsCondition> Condition; - - IDriver* Driver; + const ui64 TxId; + THolder<IEraseRowsCondition> Condition; + + IDriver* Driver; TIntrusiveConstPtr<TScheme> Scheme; - TVector<TKey> KeyOrder; - TVector<TTag> ScanTags; - TSerializedKeys SerializedKeys; - - TStats Stats; - bool NoMoreData; - bool Success; - TString Error; - -}; // TCondEraseScan - -class TIndexedCondEraseScan: public TCondEraseScan { -public: - explicit TIndexedCondEraseScan( + TVector<TKey> KeyOrder; + TVector<TTag> ScanTags; + TSerializedKeys SerializedKeys; + + TStats Stats; + bool NoMoreData; + bool Success; + TString Error; + +}; // TCondEraseScan + +class TIndexedCondEraseScan: public TCondEraseScan { +public: + explicit TIndexedCondEraseScan( TDataShard* ds, const TActorId& replyTo, const TTableId& tableId, ui64 txId, - THolder<IEraseRowsCondition> condition, const TLimits& limits, TIndexes indexes) - : TCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits) - , Indexes(std::move(indexes)) - { - } - -protected: - // Enrich key with indexed columns + THolder<IEraseRowsCondition> condition, const TLimits& limits, TIndexes indexes) + : TCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits) + , Indexes(std::move(indexes)) + { + } + +protected: + // Enrich key with indexed columns TVector<TKey> MakeKeyOrder(TIntrusiveConstPtr<IScan::TScheme> scheme) const override { - auto keyOrder = TCondEraseScan::MakeKeyOrder(scheme); - - THashSet<TTag> keys; - for (const auto& key : keyOrder) { - keys.insert(key.Tag); - } - - for (const auto& [_, keyMap] : Indexes) { + auto keyOrder = TCondEraseScan::MakeKeyOrder(scheme); + + THashSet<TTag> keys; + for (const auto& key : keyOrder) { + keys.insert(key.Tag); + } + + for (const auto& [_, keyMap] : Indexes) { for (const auto& [indexColumnId, mainColumnId] : keyMap) { - Y_UNUSED(indexColumnId); - - if (keys.contains(mainColumnId)) { - continue; - } - - const TColInfo* col = scheme->ColInfo(mainColumnId); - Y_VERIFY(col); - Y_VERIFY(col->Tag == mainColumnId); - - keyOrder.emplace_back().Tag = col->Tag; - keys.insert(col->Tag); - } - } - - return keyOrder; - } - - TActorId CreateEraser() override { - Y_VERIFY(!Eraser); - Eraser = this->Register(CreateDistributedEraser(this->SelfId(), TableId, Indexes)); - return Eraser; - } - - void CloseEraser() override { - if (!Eraser) { - return; - } - - this->Send(std::exchange(Eraser, TActorId()), new TEvents::TEvPoisonPill()); - } - -private: - const TIndexes Indexes; - - TActorId Eraser; - -}; // TIndexedCondEraseScan - -IScan* CreateCondEraseScan( + Y_UNUSED(indexColumnId); + + if (keys.contains(mainColumnId)) { + continue; + } + + const TColInfo* col = scheme->ColInfo(mainColumnId); + Y_VERIFY(col); + Y_VERIFY(col->Tag == mainColumnId); + + keyOrder.emplace_back().Tag = col->Tag; + keys.insert(col->Tag); + } + } + + return keyOrder; + } + + TActorId CreateEraser() override { + Y_VERIFY(!Eraser); + Eraser = this->Register(CreateDistributedEraser(this->SelfId(), TableId, Indexes)); + return Eraser; + } + + void CloseEraser() override { + if (!Eraser) { + return; + } + + this->Send(std::exchange(Eraser, TActorId()), new TEvents::TEvPoisonPill()); + } + +private: + const TIndexes Indexes; + + TActorId Eraser; + +}; // TIndexedCondEraseScan + +IScan* CreateCondEraseScan( TDataShard* ds, const TActorId& replyTo, const TTableId& tableId, ui64 txId, - THolder<IEraseRowsCondition> condition, const TLimits& limits, TIndexes indexes) -{ - Y_VERIFY(ds); - Y_VERIFY(condition.Get()); - - if (!indexes) { - return new TCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits); - } else { - return new TIndexedCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits, std::move(indexes)); - } -} - -static TIndexes GetIndexes(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& record) { - TIndexes result; - if (!record.IndexesSize()) { - return result; - } - - for (const auto& index : record.GetIndexes()) { - TKeyMap keyMap(Reserve(index.KeyMapSize())); - for (const auto& kv : index.GetKeyMap()) { - keyMap.emplace_back(kv.GetIndexColumnId(), kv.GetMainColumnId()); - } - - result.emplace(TTableId(index.GetOwnerId(), index.GetPathId(), index.GetSchemaVersion()), std::move(keyMap)); - } - - return result; -} - + THolder<IEraseRowsCondition> condition, const TLimits& limits, TIndexes indexes) +{ + Y_VERIFY(ds); + Y_VERIFY(condition.Get()); + + if (!indexes) { + return new TCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits); + } else { + return new TIndexedCondEraseScan(ds, replyTo, tableId, txId, std::move(condition), limits, std::move(indexes)); + } +} + +static TIndexes GetIndexes(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& record) { + TIndexes result; + if (!record.IndexesSize()) { + return result; + } + + for (const auto& index : record.GetIndexes()) { + TKeyMap keyMap(Reserve(index.KeyMapSize())); + for (const auto& kv : index.GetKeyMap()) { + keyMap.emplace_back(kv.GetIndexColumnId(), kv.GetMainColumnId()); + } + + result.emplace(TTableId(index.GetOwnerId(), index.GetPathId(), index.GetSchemaVersion()), std::move(keyMap)); + } + + return result; +} + static bool CheckUnit(NScheme::TTypeId type, NKikimrSchemeOp::TTTLSettings::EUnit unit, TString& error) { - switch (type) { - case NScheme::NTypeIds::Date: - case NScheme::NTypeIds::Datetime: - case NScheme::NTypeIds::Timestamp: + switch (type) { + case NScheme::NTypeIds::Date: + case NScheme::NTypeIds::Datetime: + case NScheme::NTypeIds::Timestamp: if (unit == NKikimrSchemeOp::TTTLSettings::UNIT_AUTO) { - return true; - } else { - error = "Unit cannot be specified for date type column"; - return false; - } - break; - - case NScheme::NTypeIds::Uint32: - case NScheme::NTypeIds::Uint64: - case NScheme::NTypeIds::DyNumber: - switch (unit) { + return true; + } else { + error = "Unit cannot be specified for date type column"; + return false; + } + break; + + case NScheme::NTypeIds::Uint32: + case NScheme::NTypeIds::Uint64: + case NScheme::NTypeIds::DyNumber: + switch (unit) { case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: - return true; + return true; case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: - error = "Unit should be specified for integral type column"; - return false; - default: - error = TStringBuilder() << "Unknown unit: " << static_cast<ui32>(unit); - return false; - } - break; - - default: - error = TStringBuilder() << "Unsupported type: " << static_cast<ui32>(type); - return false; - } -} - + error = "Unit should be specified for integral type column"; + return false; + default: + error = TStringBuilder() << "Unknown unit: " << static_cast<ui32>(unit); + return false; + } + break; + + default: + error = TStringBuilder() << "Unsupported type: " << static_cast<ui32>(type); + return false; + } +} + void TDataShard::Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, const TActorContext& ctx) { - using TEvRequest = TEvDataShard::TEvConditionalEraseRowsRequest; - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - const auto& record = ev->Get()->Record; - - auto response = MakeHolder<TEvResponse>(); - response->Record.SetTabletID(TabletID()); - - auto badRequest = [&record = response->Record](const TString& error) { - record.SetStatus(TEvResponse::ProtoRecordType::BAD_REQUEST); - record.SetErrorDescription(error); - }; - - const ui64 localPathId = record.GetTableId(); - if (GetUserTables().contains(localPathId)) { - auto condition = record.GetConditionCase(); - - if (condition && InFlightCondErase) { - if (InFlightCondErase.Condition == condition && InFlightCondErase.IsActive()) { - ctx.Send(ev->Forward(InFlightCondErase.ActorId)); - } else { - if (InFlightCondErase.Condition != condition) { - response->Record.SetStatus(TEvResponse::ProtoRecordType::OVERLOADED); - } - - ctx.Send(ev->Sender, std::move(response)); - } - - return; - } - - TUserTable::TCPtr userTable = GetUserTables().at(localPathId); - if (record.GetSchemaVersion() && userTable->GetTableSchemaVersion() - && record.GetSchemaVersion() != userTable->GetTableSchemaVersion()) { - - response->Record.SetStatus(TEvResponse::ProtoRecordType::SCHEME_ERROR); - response->Record.SetErrorDescription(TStringBuilder() << "Schema version mismatch" - << ": got " << record.GetSchemaVersion() - << ", expected " << userTable->GetTableSchemaVersion()); - ctx.Send(ev->Sender, std::move(response)); - return; - } - + using TEvRequest = TEvDataShard::TEvConditionalEraseRowsRequest; + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + const auto& record = ev->Get()->Record; + + auto response = MakeHolder<TEvResponse>(); + response->Record.SetTabletID(TabletID()); + + auto badRequest = [&record = response->Record](const TString& error) { + record.SetStatus(TEvResponse::ProtoRecordType::BAD_REQUEST); + record.SetErrorDescription(error); + }; + + const ui64 localPathId = record.GetTableId(); + if (GetUserTables().contains(localPathId)) { + auto condition = record.GetConditionCase(); + + if (condition && InFlightCondErase) { + if (InFlightCondErase.Condition == condition && InFlightCondErase.IsActive()) { + ctx.Send(ev->Forward(InFlightCondErase.ActorId)); + } else { + if (InFlightCondErase.Condition != condition) { + response->Record.SetStatus(TEvResponse::ProtoRecordType::OVERLOADED); + } + + ctx.Send(ev->Sender, std::move(response)); + } + + return; + } + + TUserTable::TCPtr userTable = GetUserTables().at(localPathId); + if (record.GetSchemaVersion() && userTable->GetTableSchemaVersion() + && record.GetSchemaVersion() != userTable->GetTableSchemaVersion()) { + + response->Record.SetStatus(TEvResponse::ProtoRecordType::SCHEME_ERROR); + response->Record.SetErrorDescription(TStringBuilder() << "Schema version mismatch" + << ": got " << record.GetSchemaVersion() + << ", expected " << userTable->GetTableSchemaVersion()); + ctx.Send(ev->Sender, std::move(response)); + return; + } + ui64 localTxId = 0; - THolder<IScan> scan; - - switch (condition) { - case TEvRequest::ProtoRecordType::kExpiration: { - const ui32 columnId = record.GetExpiration().GetColumnId(); - auto column = userTable->Columns.find(columnId); - - if (column != userTable->Columns.end()) { - TString error; - if (CheckUnit(column->second.Type, record.GetExpiration().GetColumnUnit(), error)) { - localTxId = ++NextTieBreakerIndex; - const auto tableId = TTableId(PathOwnerId, localPathId, record.GetSchemaVersion()); + THolder<IScan> scan; + + switch (condition) { + case TEvRequest::ProtoRecordType::kExpiration: { + const ui32 columnId = record.GetExpiration().GetColumnId(); + auto column = userTable->Columns.find(columnId); + + if (column != userTable->Columns.end()) { + TString error; + if (CheckUnit(column->second.Type, record.GetExpiration().GetColumnUnit(), error)) { + localTxId = ++NextTieBreakerIndex; + const auto tableId = TTableId(PathOwnerId, localPathId, record.GetSchemaVersion()); scan.Reset(CreateCondEraseScan(this, ev->Sender, tableId, localTxId, THolder(CreateEraseRowsCondition(record)), record.GetLimits(), GetIndexes(record))); - } else { - badRequest(error); - } - } else { - badRequest(TStringBuilder() << "Unknown column id: " << columnId); - } - break; - } - - default: - badRequest(TStringBuilder() << "Unknown condition: " << (ui32)condition); - break; - } - - if (scan) { - const ui32 localTableId = userTable->LocalTid; - Y_VERIFY(Executor()->Scheme().GetTableInfo(localTableId)); - - auto policy = Executor()->Scheme().GetTableInfo(localTableId)->CompactionPolicy; - auto opts = TScanOptions().SetResourceBroker(policy->BackupResourceBrokerTask, policy->DefaultTaskPriority + 2); - const ui64 scanId = QueueScan(localTableId, scan.Release(), localTxId, opts); - InFlightCondErase.Enqueue(localTxId, scanId, condition); - } - } else { - badRequest(TStringBuilder() << "Unknown table id: " << localPathId); - } - - ctx.Send(ev->Sender, std::move(response)); -} - + } else { + badRequest(error); + } + } else { + badRequest(TStringBuilder() << "Unknown column id: " << columnId); + } + break; + } + + default: + badRequest(TStringBuilder() << "Unknown condition: " << (ui32)condition); + break; + } + + if (scan) { + const ui32 localTableId = userTable->LocalTid; + Y_VERIFY(Executor()->Scheme().GetTableInfo(localTableId)); + + auto policy = Executor()->Scheme().GetTableInfo(localTableId)->CompactionPolicy; + auto opts = TScanOptions().SetResourceBroker(policy->BackupResourceBrokerTask, policy->DefaultTaskPriority + 2); + const ui64 scanId = QueueScan(localTableId, scan.Release(), localTxId, opts); + InFlightCondErase.Enqueue(localTxId, scanId, condition); + } + } else { + badRequest(TStringBuilder() << "Unknown table id: " << localPathId); + } + + ctx.Send(ev->Sender, std::move(response)); +} + void TDataShard::Handle(TEvPrivate::TEvConditionalEraseRowsRegistered::TPtr& ev, const TActorContext& ctx) { - if (!InFlightCondErase || InFlightCondErase.TxId != ev->Get()->TxId) { - LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, "Unknown conditional erase actor registered" - << ": at: " << TabletID()); - return; - } - - InFlightCondErase.ActorId = ev->Get()->ActorId; -} - + if (!InFlightCondErase || InFlightCondErase.TxId != ev->Get()->TxId) { + LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, "Unknown conditional erase actor registered" + << ": at: " << TabletID()); + return; + } + + InFlightCondErase.ActorId = ev->Get()->ActorId; +} + } // NDataShard -} // NKikimr - -Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvEraseRowsResponse::EStatus, stream, value) { - stream << NKikimrTxDataShard::TEvEraseRowsResponse_EStatus_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvConditionalEraseRowsResponse::EStatus, stream, value) { - stream << NKikimrTxDataShard::TEvConditionalEraseRowsResponse_EStatus_Name(value); -} +} // NKikimr + +Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvEraseRowsResponse::EStatus, stream, value) { + stream << NKikimrTxDataShard::TEvEraseRowsResponse_EStatus_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvConditionalEraseRowsResponse::EStatus, stream, value) { + stream << NKikimrTxDataShard::TEvConditionalEraseRowsResponse_EStatus_Name(value); +} diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index 09b3c563836..876c3370933 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -1,4 +1,4 @@ -#include "change_collector.h" +#include "change_collector.h" #include "datashard_impl.h" #include "datashard__engine_host.h" #include "sys_tables.h" @@ -278,7 +278,7 @@ TIntrusivePtr<TThrRefBase> InitDataShardSysTables(TDataShard* self) { } /// -class TDataShardEngineHost : public TEngineHost { +class TDataShardEngineHost : public TEngineHost { public: TDataShardEngineHost(TDataShard* self, NTable::TDatabase& db, TEngineHostCounters& counters, ui64& lockTxId, TInstant now) : TEngineHost(db, counters, @@ -312,40 +312,40 @@ public: return ReadVersion; } - void SetIsImmediateTx() { - IsImmediateTx = true; - } - - IChangeCollector* GetChangeCollector(const TTableId& tableId) const override { - auto it = ChangeCollectors.find(tableId); - if (it != ChangeCollectors.end()) { - return it->second.Get(); - } - - it = ChangeCollectors.emplace(tableId, nullptr).first; - if (!Self->IsUserTable(tableId)) { - return it->second.Get(); - } - - it->second.Reset(CreateChangeCollector(*Self, DB, tableId.PathId.LocalPathId, IsImmediateTx)); - return it->second.Get(); - } - - TVector<IChangeCollector::TChange> GetCollectedChanges() const { - TVector<IChangeCollector::TChange> total; - - for (auto& [_, collector] : ChangeCollectors) { - if (!collector) { - continue; - } - - auto collected = std::move(collector->GetCollected()); - std::move(collected.begin(), collected.end(), std::back_inserter(total)); - } - - return total; - } - + void SetIsImmediateTx() { + IsImmediateTx = true; + } + + IChangeCollector* GetChangeCollector(const TTableId& tableId) const override { + auto it = ChangeCollectors.find(tableId); + if (it != ChangeCollectors.end()) { + return it->second.Get(); + } + + it = ChangeCollectors.emplace(tableId, nullptr).first; + if (!Self->IsUserTable(tableId)) { + return it->second.Get(); + } + + it->second.Reset(CreateChangeCollector(*Self, DB, tableId.PathId.LocalPathId, IsImmediateTx)); + return it->second.Get(); + } + + TVector<IChangeCollector::TChange> GetCollectedChanges() const { + TVector<IChangeCollector::TChange> total; + + for (auto& [_, collector] : ChangeCollectors) { + if (!collector) { + continue; + } + + auto collected = std::move(collector->GetCollected()); + std::move(collected.begin(), collected.end(), std::back_inserter(total)); + } + + return total; + } + bool IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTime) const override { if (TSysTables::IsSystemTable(key.TableId)) return DataShardSysTable(key.TableId).IsValidKey(key); @@ -359,7 +359,7 @@ public: } NUdf::TUnboxedValue SelectRow(const TTableId& tableId, const TArrayRef<const TCell>& row, - TStructLiteral* columnIds, TOptionalType* returnType, const TReadTarget& readTarget, + TStructLiteral* columnIds, TOptionalType* returnType, const TReadTarget& readTarget, const THolderFactory& holderFactory) override { if (TSysTables::IsSystemTable(tableId)) { @@ -499,11 +499,11 @@ private: TDataShard* Self; NTable::TDatabase& DB; const ui64& LockTxId; - bool IsImmediateTx = false; + bool IsImmediateTx = false; TInstant Now; TRowVersion WriteVersion = TRowVersion::Max(); TRowVersion ReadVersion = TRowVersion::Min(); - mutable THashMap<TTableId, THolder<IChangeCollector>> ChangeCollectors; + mutable THashMap<TTableId, THolder<IChangeCollector>> ChangeCollectors; }; // @@ -656,21 +656,21 @@ void TEngineBay::SetReadVersion(TRowVersion readVersion) { ComputeCtx->SetReadVersion(readVersion); } -void TEngineBay::SetIsImmediateTx() { - Y_VERIFY(EngineHost); - +void TEngineBay::SetIsImmediateTx() { + Y_VERIFY(EngineHost); + auto* host = static_cast<TDataShardEngineHost*>(EngineHost.Get()); - host->SetIsImmediateTx(); -} - -TVector<IChangeCollector::TChange> TEngineBay::GetCollectedChanges() const { - Y_VERIFY(EngineHost); - + host->SetIsImmediateTx(); +} + +TVector<IChangeCollector::TChange> TEngineBay::GetCollectedChanges() const { + Y_VERIFY(EngineHost); + auto* host = static_cast<TDataShardEngineHost*>(EngineHost.Get()); - return host->GetCollectedChanges(); -} - -IEngineFlat * TEngineBay::GetEngine() { + return host->GetCollectedChanges(); +} + +IEngineFlat * TEngineBay::GetEngine() { if (!Engine) { Engine = CreateEngineFlat(*EngineSettings); Engine->SetStepTxId(StepTxId); @@ -710,10 +710,10 @@ NKqp::TKqpTasksRunner& TEngineBay::GetKqpTasksRunner(const NKikimrTxDataShard::T return *KqpTasksRunner; } -TKqpDatashardComputeContext& TEngineBay::GetKqpComputeCtx() { +TKqpDatashardComputeContext& TEngineBay::GetKqpComputeCtx() { Y_VERIFY(ComputeCtx); return *ComputeCtx; } -} // NDataShard -} // NKikimr +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard__engine_host.h b/ydb/core/tx/datashard/datashard__engine_host.h index afe91d80928..8e3064c83d8 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.h +++ b/ydb/core/tx/datashard/datashard__engine_host.h @@ -92,10 +92,10 @@ public: void SetWriteVersion(TRowVersion writeVersion); void SetReadVersion(TRowVersion readVersion); - void SetIsImmediateTx(); + void SetIsImmediateTx(); + + TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const; - TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const; - void ResetCounters() { EngineHostCounters = TEngineHostCounters(); } const TEngineHostCounters& GetCounters() const { return EngineHostCounters; } diff --git a/ydb/core/tx/datashard/datashard__init.cpp b/ydb/core/tx/datashard/datashard__init.cpp index 9a9a01eb892..dc12c49959b 100644 --- a/ydb/core/tx/datashard/datashard__init.cpp +++ b/ydb/core/tx/datashard/datashard__init.cpp @@ -21,18 +21,18 @@ bool TDataShard::TTxInit::Execute(TTransactionContext& txc, const TActorContext& Self->State = TShardState::Unknown; Self->LastLocalTid = Schema::MinLocalTid; Self->LastSeqno = 1; - Self->NextChangeRecordOrder = 1; - Self->LastChangeRecordGroup = 1; + Self->NextChangeRecordOrder = 1; + Self->LastChangeRecordGroup = 1; Self->TransQueue.Reset(); Self->SnapshotManager.Reset(); - Self->SchemaSnapshotManager.Reset(); - Self->S3Uploads.Reset(); - Self->S3Downloads.Reset(); - - Self->KillChangeSender(ctx); - Self->ChangesQueue.clear(); - ChangeRecords.clear(); - + Self->SchemaSnapshotManager.Reset(); + Self->S3Uploads.Reset(); + Self->S3Downloads.Reset(); + + Self->KillChangeSender(ctx); + Self->ChangesQueue.clear(); + ChangeRecords.clear(); + bool done = ReadEverything(txc); if (done && Self->State != TShardState::Offline) { @@ -102,22 +102,22 @@ void TDataShard::TTxInit::Complete(const TActorContext &ctx) { } } - Self->CreateChangeSender(ctx); - Self->EnqueueChangeRecords(std::move(ChangeRecords)); - Self->MaybeActivateChangeSender(ctx); - - if (!Self->ChangesQueue) { - if (!Self->ChangeExchangeSplitter.Done()) { - Self->ChangeExchangeSplitter.DoSplit(ctx); - } else { - for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { - if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { - Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); - } - } - } - } - + Self->CreateChangeSender(ctx); + Self->EnqueueChangeRecords(std::move(ChangeRecords)); + Self->MaybeActivateChangeSender(ctx); + + if (!Self->ChangesQueue) { + if (!Self->ChangeExchangeSplitter.Done()) { + Self->ChangeExchangeSplitter.DoSplit(ctx); + } else { + for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { + if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { + Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); + } + } + } + } + // Switch mvcc state if needed Self->CheckMvccStateChangeCanStart(ctx); } @@ -153,17 +153,17 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { PRECHARGE_SYS_TABLE(Schema::SplitSrcSnapshots); PRECHARGE_SYS_TABLE(Schema::SplitDstReceivedSnapshots); PRECHARGE_SYS_TABLE(Schema::Snapshots); - PRECHARGE_SYS_TABLE(Schema::S3Uploads); - PRECHARGE_SYS_TABLE(Schema::S3UploadedParts); - PRECHARGE_SYS_TABLE(Schema::S3Downloads); - PRECHARGE_SYS_TABLE(Schema::ChangeRecords); - PRECHARGE_SYS_TABLE(Schema::SrcChangeSenderActivations); - PRECHARGE_SYS_TABLE(Schema::DstChangeSenderActivations); + PRECHARGE_SYS_TABLE(Schema::S3Uploads); + PRECHARGE_SYS_TABLE(Schema::S3UploadedParts); + PRECHARGE_SYS_TABLE(Schema::S3Downloads); + PRECHARGE_SYS_TABLE(Schema::ChangeRecords); + PRECHARGE_SYS_TABLE(Schema::SrcChangeSenderActivations); + PRECHARGE_SYS_TABLE(Schema::DstChangeSenderActivations); PRECHARGE_SYS_TABLE(Schema::ReplicationSources); PRECHARGE_SYS_TABLE(Schema::ReplicationSourceOffsets); PRECHARGE_SYS_TABLE(Schema::DstReplicationSourceOffsetsReceived); PRECHARGE_SYS_TABLE(Schema::UserTablesStats); - PRECHARGE_SYS_TABLE(Schema::SchemaSnapshots); + PRECHARGE_SYS_TABLE(Schema::SchemaSnapshots); if (!ready) return false; @@ -175,8 +175,8 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { LOAD_SYS_UI64(db, Schema::Sys_State, Self->State); LOAD_SYS_UI64(db, Schema::Sys_LastLocalTid, Self->LastLocalTid); LOAD_SYS_UI64(db, Schema::Sys_LastSeqno, Self->LastSeqno); - LOAD_SYS_UI64(db, Schema::Sys_NextChangeRecordOrder, Self->NextChangeRecordOrder); - LOAD_SYS_UI64(db, Schema::Sys_LastChangeRecordGroup, Self->LastChangeRecordGroup); + LOAD_SYS_UI64(db, Schema::Sys_NextChangeRecordOrder, Self->NextChangeRecordOrder); + LOAD_SYS_UI64(db, Schema::Sys_LastChangeRecordGroup, Self->LastChangeRecordGroup); LOAD_SYS_UI64(db, Schema::Sys_TxReadSizeLimit, Self->TxReadSizeLimit); LOAD_SYS_UI64(db, Schema::Sys_PathOwnerId, Self->PathOwnerId); LOAD_SYS_UI64(db, Schema::Sys_CurrentSchemeShardId, Self->CurrentSchemeShardId); @@ -267,21 +267,21 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { Self->SplitSnapshotStarted = false; } - if (db.HaveTable<Schema::SrcChangeSenderActivations>()) { - // Read change sender activations on src tablet - auto rowset = db.Table<Schema::SrcChangeSenderActivations>().GreaterOrEqual(0).Select(); - if (!rowset.IsReady()) - return false; - - while (!rowset.EndOfSet()) { - ui64 dstTabletId = rowset.GetValue<Schema::SrcChangeSenderActivations::DstTabletId>(); - Self->ChangeSenderActivator.AddDst(dstTabletId); - - if (!rowset.Next()) - return false; - } - } - + if (db.HaveTable<Schema::SrcChangeSenderActivations>()) { + // Read change sender activations on src tablet + auto rowset = db.Table<Schema::SrcChangeSenderActivations>().GreaterOrEqual(0).Select(); + if (!rowset.IsReady()) + return false; + + while (!rowset.EndOfSet()) { + ui64 dstTabletId = rowset.GetValue<Schema::SrcChangeSenderActivations::DstTabletId>(); + Self->ChangeSenderActivator.AddDst(dstTabletId); + + if (!rowset.Next()) + return false; + } + } + // Split/Merge description on DST LOAD_SYS_UI64(db, Schema::Sys_DstSplitOpId, Self->DstSplitOpId); { @@ -297,8 +297,8 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { // Add all SRC datashards to the list Self->ReceiveSnapshotsFrom.clear(); - Self->ReceiveActivationsFrom.clear(); - + Self->ReceiveActivationsFrom.clear(); + if (Self->DstSplitDescription) { for (ui32 i = 0; i < Self->DstSplitDescription->SourceRangesSize(); ++i) { ui64 srcTabletId = Self->DstSplitDescription->GetSourceRanges(i).GetTabletID(); @@ -306,33 +306,33 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { } } - { - auto rowset = db.Table<Schema::SplitDstReceivedSnapshots>().GreaterOrEqual(0).Select(); - if (!rowset.IsReady()) - return false; - - // Exclude SRC datashards from which the snapshots have already been received - while (!rowset.EndOfSet()) { - ui64 srcTabletId = rowset.GetValue<Schema::SplitDstReceivedSnapshots::SrcTabletId>(); - Self->ReceiveSnapshotsFrom.erase(srcTabletId); - - if (!rowset.Next()) - return false; - } - } - - if (db.HaveTable<Schema::DstChangeSenderActivations>()) { - auto rowset = db.Table<Schema::DstChangeSenderActivations>().GreaterOrEqual(0).Select(); - if (!rowset.IsReady()) + { + auto rowset = db.Table<Schema::SplitDstReceivedSnapshots>().GreaterOrEqual(0).Select(); + if (!rowset.IsReady()) + return false; + + // Exclude SRC datashards from which the snapshots have already been received + while (!rowset.EndOfSet()) { + ui64 srcTabletId = rowset.GetValue<Schema::SplitDstReceivedSnapshots::SrcTabletId>(); + Self->ReceiveSnapshotsFrom.erase(srcTabletId); + + if (!rowset.Next()) + return false; + } + } + + if (db.HaveTable<Schema::DstChangeSenderActivations>()) { + auto rowset = db.Table<Schema::DstChangeSenderActivations>().GreaterOrEqual(0).Select(); + if (!rowset.IsReady()) return false; - - while (!rowset.EndOfSet()) { - ui64 srcTabletId = rowset.GetValue<Schema::DstChangeSenderActivations::SrcTabletId>(); - Self->ReceiveActivationsFrom.insert(srcTabletId); - - if (!rowset.Next()) - return false; - } + + while (!rowset.EndOfSet()) { + ui64 srcTabletId = rowset.GetValue<Schema::DstChangeSenderActivations::SrcTabletId>(); + Self->ReceiveActivationsFrom.insert(srcTabletId); + + if (!rowset.Next()) + return false; + } } } @@ -345,11 +345,11 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { Self->SrcSplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); bool parseOk = ParseFromStringNoSizeLimit(*Self->SrcSplitDescription, splitDescr); Y_VERIFY(parseOk); - - for (ui32 i = 0; i < Self->SrcSplitDescription->DestinationRangesSize(); ++i) { - ui64 dstTablet = Self->SrcSplitDescription->GetDestinationRanges(i).GetTabletID(); - Self->ChangeExchangeSplitter.AddDst(dstTablet); - } + + for (ui32 i = 0; i < Self->SrcSplitDescription->DestinationRangesSize(); ++i) { + ui64 dstTablet = Self->SrcSplitDescription->GetDestinationRanges(i).GetTabletID(); + Self->ChangeExchangeSplitter.AddDst(dstTablet); + } } } @@ -388,22 +388,22 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { return false; } - if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::S3Uploads::TableId)) { - if (!Self->S3Uploads.Load(db)) - return false; - } - - if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::S3Downloads::TableId)) { - if (!Self->S3Downloads.Load(db)) - return false; - } - - if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::ChangeRecords::TableId)) { + if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::S3Uploads::TableId)) { + if (!Self->S3Uploads.Load(db)) + return false; + } + + if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::S3Downloads::TableId)) { + if (!Self->S3Downloads.Load(db)) + return false; + } + + if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::ChangeRecords::TableId)) { if (!Self->LoadChangeRecords(db, ChangeRecords)) { - return false; - } - } - + return false; + } + } + Self->ReplicatedTables.clear(); if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::ReplicationSources::TableId)) { auto rowset = db.Table<Schema::ReplicationSources>().Select(); @@ -473,12 +473,12 @@ bool TDataShard::TTxInit::ReadEverything(TTransactionContext &txc) { } } - if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::SchemaSnapshots::TableId)) { - if (!Self->SchemaSnapshotManager.Load(db)) { - return false; - } - } - + if (Self->State != TShardState::Offline && txc.DB.GetScheme().GetTableInfo(Schema::SchemaSnapshots::TableId)) { + if (!Self->SchemaSnapshotManager.Load(db)) { + return false; + } + } + return true; } @@ -508,9 +508,9 @@ public: // Skip full schema migration (and dropped system table recreation) // if the datashard is in the process of drop. if (Self->State == TShardState::PreOffline || Self->State == TShardState::Offline) { - db.MaterializeExisting<Schema>(); - Schema::SchemaTables<Schema::SchemaOperations>::Materialize(txc.DB, NIceDb::EMaterializationMode::NonExisting); - Schema::SchemaTables<Schema::ScanProgress>::Materialize(txc.DB, NIceDb::EMaterializationMode::NonExisting); + db.MaterializeExisting<Schema>(); + Schema::SchemaTables<Schema::SchemaOperations>::Materialize(txc.DB, NIceDb::EMaterializationMode::NonExisting); + Schema::SchemaTables<Schema::ScanProgress>::Materialize(txc.DB, NIceDb::EMaterializationMode::NonExisting); } else { db.Materialize<Schema>(); } @@ -521,12 +521,12 @@ public: Self->PersistSys(db, Schema::Sys_State, Self->State); - if (AppData(ctx)->FeatureFlags.GetEnableMvcc()) { - auto state = *AppData(ctx)->FeatureFlags.GetEnableMvcc() ? EMvccState::MvccEnabled : EMvccState::MvccDisabled; + if (AppData(ctx)->FeatureFlags.GetEnableMvcc()) { + auto state = *AppData(ctx)->FeatureFlags.GetEnableMvcc() ? EMvccState::MvccEnabled : EMvccState::MvccDisabled; Self->PersistSys(db, Schema::SysMvcc_State, (ui32)state); LOG_DEBUG(ctx, NKikimrServices::TX_DATASHARD, TStringBuilder() << "TxInitSchema.Execute" - << " MVCC state switched to" << (*AppData(ctx)->FeatureFlags.GetEnableMvcc() ? " enabled" : " disabled") << " state"); + << " MVCC state switched to" << (*AppData(ctx)->FeatureFlags.GetEnableMvcc() ? " enabled" : " disabled") << " state"); } Self->MvccSwitchState = TSwitchState::DONE; diff --git a/ydb/core/tx/datashard/datashard__op_rows.cpp b/ydb/core/tx/datashard/datashard__op_rows.cpp index e0181a41995..3b3c9d9d6f9 100644 --- a/ydb/core/tx/datashard/datashard__op_rows.cpp +++ b/ydb/core/tx/datashard/datashard__op_rows.cpp @@ -6,9 +6,9 @@ namespace NDataShard { using namespace NTabletFlatExecutor; -template <typename TEvRequest> +template <typename TEvRequest> class TTxDirectBase : public TTransactionBase<TDataShard> { - TEvRequest Ev; + TEvRequest Ev; TOperation::TPtr Op; TVector<EExecutionUnitKind> CompleteList; @@ -17,13 +17,13 @@ public: TTxDirectBase(TDataShard* ds, TEvRequest ev) : TBase(ds) , Ev(ev) - { - } + { + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxDirectBase(" << GetTxType() << ") Execute" + << ": at tablet# " << Self->TabletID()); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxDirectBase(" << GetTxType() << ") Execute" - << ": at tablet# " << Self->TabletID()); - if (Self->IsFollower()) { return true; // TODO: report error } @@ -48,48 +48,48 @@ public: } } - void Complete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxDirectBase(" << GetTxType() << ") Complete" - << ": at tablet# " << Self->TabletID()); - + void Complete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxDirectBase(" << GetTxType() << ") Complete" + << ": at tablet# " << Self->TabletID()); + Self->Pipeline.RunCompleteList(Op, CompleteList, ctx); - if (Self->Pipeline.CanRunAnotherOp()) { + if (Self->Pipeline.CanRunAnotherOp()) { Self->PlanQueue.Progress(ctx); - } + } } - -}; // TTxDirectBase - + +}; // TTxDirectBase + class TDataShard::TTxUploadRows : public TTxDirectBase<TEvDataShard::TEvUploadRowsRequest::TPtr> { -public: - using TTxDirectBase::TTxDirectBase; - TTxType GetTxType() const override { return TXTYPE_UPLOAD_ROWS; } +public: + using TTxDirectBase::TTxDirectBase; + TTxType GetTxType() const override { return TXTYPE_UPLOAD_ROWS; } }; class TDataShard::TTxEraseRows : public TTxDirectBase<TEvDataShard::TEvEraseRowsRequest::TPtr> { -public: - using TTxDirectBase::TTxDirectBase; - TTxType GetTxType() const override { return TXTYPE_ERASE_ROWS; } -}; - +public: + using TTxDirectBase::TTxDirectBase; + TTxType GetTxType() const override { return TXTYPE_ERASE_ROWS; } +}; + static void OutOfSpace(NKikimrTxDataShard::TEvUploadRowsResponse& response) { response.SetStatus(NKikimrTxDataShard::TError::OUT_OF_SPACE); } -static void WrongShardState(NKikimrTxDataShard::TEvUploadRowsResponse& response) { - response.SetStatus(NKikimrTxDataShard::TError::WRONG_SHARD_STATE); -} - +static void WrongShardState(NKikimrTxDataShard::TEvUploadRowsResponse& response) { + response.SetStatus(NKikimrTxDataShard::TError::WRONG_SHARD_STATE); +} + static void OutOfSpace(NKikimrTxDataShard::TEvEraseRowsResponse& response) { // NOTE: this function is never called, because erase is allowed when out of space response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::WRONG_SHARD_STATE); } -static void WrongShardState(NKikimrTxDataShard::TEvEraseRowsResponse& response) { - response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::WRONG_SHARD_STATE); -} - -template <typename TEvResponse, typename TEvRequest> +static void WrongShardState(NKikimrTxDataShard::TEvEraseRowsResponse& response) { + response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::WRONG_SHARD_STATE); +} + +template <typename TEvResponse, typename TEvRequest> static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& ctx, const TString& txDesc, bool isWrite) { NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus; TString rejectReason; @@ -110,42 +110,42 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c } } - if (!reject) { - return false; - } + if (!reject) { + return false; + } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard" - << ": tablet# " << self->TabletID() + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard" + << ": tablet# " << self->TabletID() << ", error# " << rejectReason); - auto response = MakeHolder<TEvResponse>(); - response->Record.SetTabletID(self->TabletID()); + auto response = MakeHolder<TEvResponse>(); + response->Record.SetTabletID(self->TabletID()); if (outOfSpace) { OutOfSpace(response->Record); } else { WrongShardState(response->Record); } response->Record.SetErrorDescription(rejectReason); - ctx.Send(ev->Sender, std::move(response)); + ctx.Send(ev->Sender, std::move(response)); + + return true; +} - return true; -} - void TDataShard::Handle(TEvDataShard::TEvUploadRowsRequest::TPtr& ev, const TActorContext& ctx) { if (!MaybeReject<TEvDataShard::TEvUploadRowsResponse>(this, ev, ctx, "bulk upsert", true)) { - Executor()->Execute(new TTxUploadRows(this, ev), ctx); - } else { + Executor()->Execute(new TTxUploadRows(this, ev), ctx); + } else { IncCounter(COUNTER_BULK_UPSERT_OVERLOADED); } } void TDataShard::Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActorContext& ctx) { if (!MaybeReject<TEvDataShard::TEvEraseRowsResponse>(this, ev, ctx, "erase", false)) { - Executor()->Execute(new TTxEraseRows(this, ev), ctx); - } else { - IncCounter(COUNTER_ERASE_ROWS_OVERLOADED); - } -} - + Executor()->Execute(new TTxEraseRows(this, ev), ctx); + } else { + IncCounter(COUNTER_ERASE_ROWS_OVERLOADED); + } +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard__s3_download_txs.cpp b/ydb/core/tx/datashard/datashard__s3_download_txs.cpp index 0eb58119014..e726171a8ef 100644 --- a/ydb/core/tx/datashard/datashard__s3_download_txs.cpp +++ b/ydb/core/tx/datashard/datashard__s3_download_txs.cpp @@ -1,60 +1,60 @@ -#include "datashard_txs.h" - -namespace NKikimr { +#include "datashard_txs.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - -/// Get + +using namespace NTabletFlatExecutor; + +/// Get TDataShard::TTxGetS3DownloadInfo::TTxGetS3DownloadInfo( TDataShard* ds, - TEvDataShard::TEvGetS3DownloadInfo::TPtr ev) - : TBase(ds) - , Ev(std::move(ev)) -{ } - + TEvDataShard::TEvGetS3DownloadInfo::TPtr ev) + : TBase(ds) + , Ev(std::move(ev)) +{ } + bool TDataShard::TTxGetS3DownloadInfo::Execute(TTransactionContext& txc, const TActorContext&) { - txc.DB.NoMoreReadsForTx(); - - const auto* info = Self->S3Downloads.Find(Ev->Get()->TxId); - if (!info) { - Reply.Reset(new TEvDataShard::TEvS3DownloadInfo); - } else { - Reply.Reset(new TEvDataShard::TEvS3DownloadInfo(*info)); - } - - return true; -} - + txc.DB.NoMoreReadsForTx(); + + const auto* info = Self->S3Downloads.Find(Ev->Get()->TxId); + if (!info) { + Reply.Reset(new TEvDataShard::TEvS3DownloadInfo); + } else { + Reply.Reset(new TEvDataShard::TEvS3DownloadInfo(*info)); + } + + return true; +} + void TDataShard::TTxGetS3DownloadInfo::Complete(const TActorContext& ctx) { - Y_VERIFY(Reply); - ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); -} - -/// Store + Y_VERIFY(Reply); + ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); +} + +/// Store TDataShard::TTxStoreS3DownloadInfo::TTxStoreS3DownloadInfo( TDataShard* ds, - TEvDataShard::TEvStoreS3DownloadInfo::TPtr ev) - : TBase(ds) - , Ev(std::move(ev)) -{ } - + TEvDataShard::TEvStoreS3DownloadInfo::TPtr ev) + : TBase(ds) + , Ev(std::move(ev)) +{ } + bool TDataShard::TTxStoreS3DownloadInfo::Execute(TTransactionContext& txc, const TActorContext&) { - const auto& msg = *Ev->Get(); - - txc.DB.NoMoreReadsForTx(); - NIceDb::TNiceDb db(txc.DB); - - const auto& info = Self->S3Downloads.Store(db, msg); - Reply.Reset(new TEvDataShard::TEvS3DownloadInfo(info)); - - return true; -} - + const auto& msg = *Ev->Get(); + + txc.DB.NoMoreReadsForTx(); + NIceDb::TNiceDb db(txc.DB); + + const auto& info = Self->S3Downloads.Store(db, msg); + Reply.Reset(new TEvDataShard::TEvS3DownloadInfo(info)); + + return true; +} + void TDataShard::TTxStoreS3DownloadInfo::Complete(const TActorContext& ctx) { - Y_VERIFY(Reply); - ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); -} - + Y_VERIFY(Reply); + ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard__s3_upload_txs.cpp b/ydb/core/tx/datashard/datashard__s3_upload_txs.cpp index 9f9e7f23692..4d9db1f74cd 100644 --- a/ydb/core/tx/datashard/datashard__s3_upload_txs.cpp +++ b/ydb/core/tx/datashard/datashard__s3_upload_txs.cpp @@ -1,80 +1,80 @@ -#include "datashard_txs.h" - -namespace NKikimr { +#include "datashard_txs.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - -/// Get + +using namespace NTabletFlatExecutor; + +/// Get TDataShard::TTxGetS3Upload::TTxGetS3Upload( TDataShard* ds, - TEvDataShard::TEvGetS3Upload::TPtr ev) - : TBase(ds) - , Ev(std::move(ev)) -{ } - + TEvDataShard::TEvGetS3Upload::TPtr ev) + : TBase(ds) + , Ev(std::move(ev)) +{ } + bool TDataShard::TTxGetS3Upload::Execute(TTransactionContext& txc, const TActorContext&) { - txc.DB.NoMoreReadsForTx(); - - if (const auto* upload = Self->S3Uploads.Find(Ev->Get()->TxId)) { - Reply.Reset(new TEvDataShard::TEvS3Upload(*upload)); - } else { - Reply.Reset(new TEvDataShard::TEvS3Upload); - } - - return true; -} - + txc.DB.NoMoreReadsForTx(); + + if (const auto* upload = Self->S3Uploads.Find(Ev->Get()->TxId)) { + Reply.Reset(new TEvDataShard::TEvS3Upload(*upload)); + } else { + Reply.Reset(new TEvDataShard::TEvS3Upload); + } + + return true; +} + void TDataShard::TTxGetS3Upload::Complete(const TActorContext& ctx) { - Y_VERIFY(Reply); - ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); -} - -/// Store id + Y_VERIFY(Reply); + ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); +} + +/// Store id TDataShard::TTxStoreS3UploadId::TTxStoreS3UploadId( TDataShard* ds, - TEvDataShard::TEvStoreS3UploadId::TPtr ev) - : TBase(ds) - , Ev(std::move(ev)) -{ } - + TEvDataShard::TEvStoreS3UploadId::TPtr ev) + : TBase(ds) + , Ev(std::move(ev)) +{ } + bool TDataShard::TTxStoreS3UploadId::Execute(TTransactionContext& txc, const TActorContext&) { - txc.DB.NoMoreReadsForTx(); - - const auto& msg = *Ev->Get(); - const auto& upload = Self->S3Uploads.Add(txc.DB, msg.TxId, msg.UploadId); - Reply.Reset(new TEvDataShard::TEvS3Upload(upload)); - - return true; -} - + txc.DB.NoMoreReadsForTx(); + + const auto& msg = *Ev->Get(); + const auto& upload = Self->S3Uploads.Add(txc.DB, msg.TxId, msg.UploadId); + Reply.Reset(new TEvDataShard::TEvS3Upload(upload)); + + return true; +} + void TDataShard::TTxStoreS3UploadId::Complete(const TActorContext& ctx) { - Y_VERIFY(Reply); - ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); -} - -/// Change status + Y_VERIFY(Reply); + ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); +} + +/// Change status TDataShard::TTxChangeS3UploadStatus::TTxChangeS3UploadStatus( TDataShard* ds, - TEvDataShard::TEvChangeS3UploadStatus::TPtr ev) - : TBase(ds) - , Ev(std::move(ev)) -{ } - + TEvDataShard::TEvChangeS3UploadStatus::TPtr ev) + : TBase(ds) + , Ev(std::move(ev)) +{ } + bool TDataShard::TTxChangeS3UploadStatus::Execute(TTransactionContext& txc, const TActorContext&) { - txc.DB.NoMoreReadsForTx(); - - auto& msg = *Ev->Get(); - const auto& upload = Self->S3Uploads.ChangeStatus(txc.DB, msg.TxId, msg.Status, std::move(msg.Error), std::move(msg.Parts)); - Reply.Reset(new TEvDataShard::TEvS3Upload(upload)); - - return true; -} - + txc.DB.NoMoreReadsForTx(); + + auto& msg = *Ev->Get(); + const auto& upload = Self->S3Uploads.ChangeStatus(txc.DB, msg.TxId, msg.Status, std::move(msg.Error), std::move(msg.Parts)); + Reply.Reset(new TEvDataShard::TEvS3Upload(upload)); + + return true; +} + void TDataShard::TTxChangeS3UploadStatus::Complete(const TActorContext& ctx) { - Y_VERIFY(Reply); - ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); -} - + Y_VERIFY(Reply); + ctx.Send(Ev->Get()->ReplyTo, Reply.Release(), 0, Ev->Cookie); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp index 7b4e53a7c71..0226bd42e25 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.cpp +++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp @@ -47,9 +47,9 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self, if (Tx.GetLockTxId()) EngineBay.SetLockTxId(Tx.GetLockTxId()); - if (Tx.GetImmediate()) - EngineBay.SetIsImmediateTx(); - + if (Tx.GetImmediate()) + EngineBay.SetIsImmediateTx(); + if (Tx.HasReadTableTransaction()) { auto &tx = Tx.GetReadTableTransaction(); if (self->TableInfos.contains(tx.GetTableId().GetTableId())) { @@ -401,8 +401,8 @@ void TActiveTransaction::FillTxData(TDataShard *self, BuildSchemeTx(); } else if (IsSnapshotTx()) { BuildSnapshotTx(); - } else if (IsDistributedEraseTx()) { - BuildDistributedEraseTx(); + } else if (IsDistributedEraseTx()) { + BuildDistributedEraseTx(); } else if (IsCommitWritesTx()) { BuildCommitWritesTx(); } @@ -437,54 +437,54 @@ bool TActiveTransaction::BuildSchemeTx() + (ui32)SchemeTx->HasDropTable() + (ui32)SchemeTx->HasAlterTable() + (ui32)SchemeTx->HasBackup() - + (ui32)SchemeTx->HasRestore() + + (ui32)SchemeTx->HasRestore() + (ui32)SchemeTx->HasSendSnapshot() + (ui32)SchemeTx->HasCreatePersistentSnapshot() + (ui32)SchemeTx->HasDropPersistentSnapshot() + (ui32)SchemeTx->HasInitiateBuildIndex() + (ui32)SchemeTx->HasFinalizeBuildIndex() + (ui32)SchemeTx->HasDropIndexNotice() - + (ui32)SchemeTx->HasMoveTable() - + (ui32)SchemeTx->HasCreateCdcStreamNotice() - + (ui32)SchemeTx->HasAlterCdcStreamNotice() - + (ui32)SchemeTx->HasDropCdcStreamNotice(); + + (ui32)SchemeTx->HasMoveTable() + + (ui32)SchemeTx->HasCreateCdcStreamNotice() + + (ui32)SchemeTx->HasAlterCdcStreamNotice() + + (ui32)SchemeTx->HasDropCdcStreamNotice(); if (count != 1) return false; if (SchemeTx->HasCreateTable()) - SchemeTxType = TSchemaOperation::ETypeCreate; + SchemeTxType = TSchemaOperation::ETypeCreate; else if (SchemeTx->HasDropTable()) - SchemeTxType = TSchemaOperation::ETypeDrop; + SchemeTxType = TSchemaOperation::ETypeDrop; else if (SchemeTx->HasAlterTable()) - SchemeTxType = TSchemaOperation::ETypeAlter; + SchemeTxType = TSchemaOperation::ETypeAlter; else if (SchemeTx->HasBackup()) - SchemeTxType = TSchemaOperation::ETypeBackup; - else if (SchemeTx->HasRestore()) - SchemeTxType = TSchemaOperation::ETypeRestore; + SchemeTxType = TSchemaOperation::ETypeBackup; + else if (SchemeTx->HasRestore()) + SchemeTxType = TSchemaOperation::ETypeRestore; else if (SchemeTx->HasSendSnapshot()) - SchemeTxType = TSchemaOperation::ETypeCopy; + SchemeTxType = TSchemaOperation::ETypeCopy; else if (SchemeTx->HasCreatePersistentSnapshot()) - SchemeTxType = TSchemaOperation::ETypeCreatePersistentSnapshot; + SchemeTxType = TSchemaOperation::ETypeCreatePersistentSnapshot; else if (SchemeTx->HasDropPersistentSnapshot()) - SchemeTxType = TSchemaOperation::ETypeDropPersistentSnapshot; + SchemeTxType = TSchemaOperation::ETypeDropPersistentSnapshot; else if (SchemeTx->HasInitiateBuildIndex()) - SchemeTxType = TSchemaOperation::ETypeInitiateBuildIndex; + SchemeTxType = TSchemaOperation::ETypeInitiateBuildIndex; else if (SchemeTx->HasFinalizeBuildIndex()) - SchemeTxType = TSchemaOperation::ETypeFinalizeBuildIndex; + SchemeTxType = TSchemaOperation::ETypeFinalizeBuildIndex; else if (SchemeTx->HasDropIndexNotice()) - SchemeTxType = TSchemaOperation::ETypeDropIndexNotice; + SchemeTxType = TSchemaOperation::ETypeDropIndexNotice; else if (SchemeTx->HasMoveTable()) SchemeTxType = TSchemaOperation::ETypeMoveTable; - else if (SchemeTx->HasCreateCdcStreamNotice()) - SchemeTxType = TSchemaOperation::ETypeCreateCdcStream; - else if (SchemeTx->HasAlterCdcStreamNotice()) - SchemeTxType = TSchemaOperation::ETypeAlterCdcStream; - else if (SchemeTx->HasDropCdcStreamNotice()) - SchemeTxType = TSchemaOperation::ETypeDropCdcStream; + else if (SchemeTx->HasCreateCdcStreamNotice()) + SchemeTxType = TSchemaOperation::ETypeCreateCdcStream; + else if (SchemeTx->HasAlterCdcStreamNotice()) + SchemeTxType = TSchemaOperation::ETypeAlterCdcStream; + else if (SchemeTx->HasDropCdcStreamNotice()) + SchemeTxType = TSchemaOperation::ETypeDropCdcStream; else - SchemeTxType = TSchemaOperation::ETypeUnknown; + SchemeTxType = TSchemaOperation::ETypeUnknown; - return SchemeTxType != TSchemaOperation::ETypeUnknown; + return SchemeTxType != TSchemaOperation::ETypeUnknown; } bool TActiveTransaction::BuildSnapshotTx() @@ -505,20 +505,20 @@ bool TActiveTransaction::BuildSnapshotTx() return true; } -bool TDistributedEraseTx::TryParse(const TString& serialized) { - if (!Body.ParseFromArray(serialized.data(), serialized.size())) { - return false; - } - - return true; -} - -bool TActiveTransaction::BuildDistributedEraseTx() { - Y_VERIFY(TxBody); +bool TDistributedEraseTx::TryParse(const TString& serialized) { + if (!Body.ParseFromArray(serialized.data(), serialized.size())) { + return false; + } + + return true; +} + +bool TActiveTransaction::BuildDistributedEraseTx() { + Y_VERIFY(TxBody); DistributedEraseTx.Reset(new TDistributedEraseTx); - return DistributedEraseTx->TryParse(TxBody); -} - + return DistributedEraseTx->TryParse(TxBody); +} + // bool TCommitWritesTx::TryParse(const TString& serialized) { @@ -806,25 +806,25 @@ void TActiveTransaction::BuildExecutionPlan(bool loaded) plan.push_back(EExecutionUnitKind::CompleteOperation); } plan.push_back(EExecutionUnitKind::CompletedOperations); - } else if (IsDistributedEraseTx()) { - if (!loaded) { - plan.push_back(EExecutionUnitKind::CheckDistributedEraseTx); - plan.push_back(EExecutionUnitKind::StoreDistributedEraseTx); - plan.push_back(EExecutionUnitKind::FinishPropose); - } - if (!GetStep()) { - plan.push_back(EExecutionUnitKind::WaitForPlan); - } - plan.push_back(EExecutionUnitKind::PlanQueue); - plan.push_back(EExecutionUnitKind::LoadTxDetails); - plan.push_back(EExecutionUnitKind::BuildAndWaitDependencies); - plan.push_back(EExecutionUnitKind::BuildDistributedEraseTxOutRS); - plan.push_back(EExecutionUnitKind::StoreAndSendOutRS); - plan.push_back(EExecutionUnitKind::PrepareDistributedEraseTxInRS); - plan.push_back(EExecutionUnitKind::LoadAndWaitInRS); - plan.push_back(EExecutionUnitKind::ExecuteDistributedEraseTx); - plan.push_back(EExecutionUnitKind::CompleteOperation); - plan.push_back(EExecutionUnitKind::CompletedOperations); + } else if (IsDistributedEraseTx()) { + if (!loaded) { + plan.push_back(EExecutionUnitKind::CheckDistributedEraseTx); + plan.push_back(EExecutionUnitKind::StoreDistributedEraseTx); + plan.push_back(EExecutionUnitKind::FinishPropose); + } + if (!GetStep()) { + plan.push_back(EExecutionUnitKind::WaitForPlan); + } + plan.push_back(EExecutionUnitKind::PlanQueue); + plan.push_back(EExecutionUnitKind::LoadTxDetails); + plan.push_back(EExecutionUnitKind::BuildAndWaitDependencies); + plan.push_back(EExecutionUnitKind::BuildDistributedEraseTxOutRS); + plan.push_back(EExecutionUnitKind::StoreAndSendOutRS); + plan.push_back(EExecutionUnitKind::PrepareDistributedEraseTxInRS); + plan.push_back(EExecutionUnitKind::LoadAndWaitInRS); + plan.push_back(EExecutionUnitKind::ExecuteDistributedEraseTx); + plan.push_back(EExecutionUnitKind::CompleteOperation); + plan.push_back(EExecutionUnitKind::CompletedOperations); } else if (IsCommitWritesTx()) { if (IsImmediate()) { Y_VERIFY(!loaded); @@ -865,8 +865,8 @@ void TActiveTransaction::BuildExecutionPlan(bool loaded) plan.push_back(EExecutionUnitKind::StoreAndSendOutRS); plan.push_back(EExecutionUnitKind::PrepareSchemeTxInRS); plan.push_back(EExecutionUnitKind::LoadAndWaitInRS); - plan.push_back(EExecutionUnitKind::Backup); - plan.push_back(EExecutionUnitKind::Restore); + plan.push_back(EExecutionUnitKind::Backup); + plan.push_back(EExecutionUnitKind::Restore); plan.push_back(EExecutionUnitKind::CreateTable); plan.push_back(EExecutionUnitKind::ReceiveSnapshot); plan.push_back(EExecutionUnitKind::AlterMoveShadow); @@ -878,9 +878,9 @@ void TActiveTransaction::BuildExecutionPlan(bool loaded) plan.push_back(EExecutionUnitKind::FinalizeBuildIndex); plan.push_back(EExecutionUnitKind::DropIndexNotice); plan.push_back(EExecutionUnitKind::MoveTable); - plan.push_back(EExecutionUnitKind::CreateCdcStream); - plan.push_back(EExecutionUnitKind::AlterCdcStream); - plan.push_back(EExecutionUnitKind::DropCdcStream); + plan.push_back(EExecutionUnitKind::CreateCdcStream); + plan.push_back(EExecutionUnitKind::AlterCdcStream); + plan.push_back(EExecutionUnitKind::DropCdcStream); plan.push_back(EExecutionUnitKind::CompleteOperation); plan.push_back(EExecutionUnitKind::CompletedOperations); } else { @@ -903,15 +903,15 @@ void TActiveTransaction::FillState(NKikimrTxDataShard::TEvGetOperationResponse & } } -void TActiveTransaction::KillAsyncJobActor(const TActorContext& ctx) { - if (!GetAsyncJobActor()) { - return; - } - - ctx.Send(GetAsyncJobActor(), new TEvents::TEvPoison()); - SetAsyncJobActor(TActorId()); -} - +void TActiveTransaction::KillAsyncJobActor(const TActorContext& ctx) { + if (!GetAsyncJobActor()) { + return; + } + + ctx.Send(GetAsyncJobActor(), new TEvents::TEvPoison()); + SetAsyncJobActor(TActorId()); +} + void TActiveTransaction::TrackMemory() const { NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Add(TxBody.size()); } diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index ffa9b0d277c..43abd525e15 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -34,7 +34,7 @@ struct TScanState { NYql::TIssues Issues; }; -struct TSchemaOperation { +struct TSchemaOperation { enum EType : ui32 { ETypeDrop = 0, ETypeCreate = 1, @@ -47,11 +47,11 @@ struct TSchemaOperation { ETypeInitiateBuildIndex = 8, ETypeFinalizeBuildIndex = 9, ETypeDropIndexNotice = 10, - ETypeRestore = 11, + ETypeRestore = 11, ETypeMoveTable = 12, - ETypeCreateCdcStream = 13, - ETypeAlterCdcStream = 14, - ETypeDropCdcStream = 15, + ETypeCreateCdcStream = 13, + ETypeAlterCdcStream = 14, + ETypeDropCdcStream = 15, ETypeUnknown = Max<ui32>() }; @@ -67,15 +67,15 @@ struct TSchemaOperation { bool Done; bool Success; - TString Error; - ui64 BytesProcessed; - ui64 RowsProcessed; + TString Error; + ui64 BytesProcessed; + ui64 RowsProcessed; TScanState ScanState; - TSchemaOperation(ui64 txId, EType type, TActorId source, ui64 tabletId, - ui64 minStep, ui64 maxStep, ui64 planStep, bool readOnly, - bool success, const TString& error, ui64 bytes, ui64 rows) + TSchemaOperation(ui64 txId, EType type, TActorId source, ui64 tabletId, + ui64 minStep, ui64 maxStep, ui64 planStep, bool readOnly, + bool success, const TString& error, ui64 bytes, ui64 rows) : TxId(txId) , Type(type) , Source(source) @@ -85,17 +85,17 @@ struct TSchemaOperation { , PlanStep(planStep) , ReadOnly(readOnly) , Done(false) - , Success(success) - , Error(error) - , BytesProcessed(bytes) - , RowsProcessed(rows) + , Success(success) + , Error(error) + , BytesProcessed(bytes) + , RowsProcessed(rows) {} bool IsDrop() const { return Type == ETypeDrop; } bool IsCreate() const { return Type == ETypeCreate; } bool IsAlter() const { return Type == ETypeAlter; } bool IsBackup() const { return Type == ETypeBackup; } - bool IsRestore() const { return Type == ETypeRestore; } + bool IsRestore() const { return Type == ETypeRestore; } bool IsCopy() const { return Type == ETypeCopy; } bool IsCreatePersistentSnapshot() const { return Type == ETypeCreatePersistentSnapshot; } bool IsDropPersistentSnapshot() const { return Type == ETypeDropPersistentSnapshot; } @@ -103,9 +103,9 @@ struct TSchemaOperation { bool IsFinalizeBuildIndex() const { return Type == ETypeFinalizeBuildIndex; } bool IsDropIndexNotice() const { return Type == ETypeDropIndexNotice; } bool IsMove() const { return Type == ETypeMoveTable; } - bool IsCreateCdcStream() const { return Type == ETypeCreateCdcStream; } - bool IsAlterCdcStream() const { return Type == ETypeAlterCdcStream; } - bool IsDropCdcStream() const { return Type == ETypeDropCdcStream; } + bool IsCreateCdcStream() const { return Type == ETypeCreateCdcStream; } + bool IsAlterCdcStream() const { return Type == ETypeAlterCdcStream; } + bool IsDropCdcStream() const { return Type == ETypeDropCdcStream; } bool IsReadOnly() const { return ReadOnly; } }; @@ -176,8 +176,8 @@ public: void SetWriteVersion(TRowVersion writeVersion) { EngineBay.SetWriteVersion(writeVersion); } void SetReadVersion(TRowVersion readVersion) { EngineBay.SetReadVersion(readVersion); } - TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const { return EngineBay.GetCollectedChanges(); } - + TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const { return EngineBay.GetCollectedChanges(); } + TActorId Source() const { return Source_; } void SetSource(const TActorId& actorId) { Source_ = actorId; } void SetStep(ui64 step) { StepTxId_.Step = step; } @@ -256,37 +256,37 @@ enum class ERestoreDataStatus { }; /// -class TDistributedEraseTx { -public: - using TPtr = THolder<TDistributedEraseTx>; - using TProto = NKikimrTxDataShard::TDistributedEraseTransaction; - -public: - bool TryParse(const TString& serialized); - - const TProto& GetBody() const { return Body; } - - bool HasDependents() const { return Body.DependentsSize(); } - const auto& GetDependents() const { return Body.GetDependents(); } - - bool HasDependencies() const { return Body.DependenciesSize(); } - const auto& GetDependencies() const { return Body.GetDependencies(); } - - const auto& GetRequest() const { return Body.GetEraseRowsRequest(); } - - const auto& GetIndexColumnIds() const { return Body.GetIndexColumnIds(); } - const auto& GetIndexColumns() const { return Body.GetIndexColumns(); } - - void SetConfirmedRows(TDynBitMap&& confirmedRows) { ConfirmedRows = std::move(confirmedRows); } - const TDynBitMap& GetConfirmedRows() const { return ConfirmedRows; } - -private: - TProto Body; - TDynBitMap ConfirmedRows; // available on shard of main table - -}; // TDistributedEraseTx - -/// +class TDistributedEraseTx { +public: + using TPtr = THolder<TDistributedEraseTx>; + using TProto = NKikimrTxDataShard::TDistributedEraseTransaction; + +public: + bool TryParse(const TString& serialized); + + const TProto& GetBody() const { return Body; } + + bool HasDependents() const { return Body.DependentsSize(); } + const auto& GetDependents() const { return Body.GetDependents(); } + + bool HasDependencies() const { return Body.DependenciesSize(); } + const auto& GetDependencies() const { return Body.GetDependencies(); } + + const auto& GetRequest() const { return Body.GetEraseRowsRequest(); } + + const auto& GetIndexColumnIds() const { return Body.GetIndexColumnIds(); } + const auto& GetIndexColumns() const { return Body.GetIndexColumns(); } + + void SetConfirmedRows(TDynBitMap&& confirmedRows) { ConfirmedRows = std::move(confirmedRows); } + const TDynBitMap& GetConfirmedRows() const { return ConfirmedRows; } + +private: + TProto Body; + TDynBitMap ConfirmedRows; // available on shard of main table + +}; // TDistributedEraseTx + +/// class TCommitWritesTx { public: using TPtr = THolder<TCommitWritesTx>; @@ -318,7 +318,7 @@ public: , ReleasedTxDataSize(0) , SchemeShardId(0) , SubDomainPathId(0) - , SchemeTxType(TSchemaOperation::ETypeUnknown) + , SchemeTxType(TSchemaOperation::ETypeUnknown) , ScanSnapshotId(0) , ScanTask(0) { @@ -393,7 +393,7 @@ public: } bool BuildSchemeTx(); void ClearSchemeTx() { SchemeTx = nullptr; } - TSchemaOperation::EType GetSchemeTxType() const { return SchemeTxType; } + TSchemaOperation::EType GetSchemeTxType() const { return SchemeTxType; } const NKikimrTxDataShard::TSnapshotTransaction& GetSnapshotTx() const { Y_VERIFY_DEBUG(SnapshotTx); @@ -402,13 +402,13 @@ public: bool BuildSnapshotTx(); void ClearSnapshotTx() { SnapshotTx = nullptr; } - const TDistributedEraseTx::TPtr& GetDistributedEraseTx() const { - Y_VERIFY_DEBUG(DistributedEraseTx); - return DistributedEraseTx; - } - bool BuildDistributedEraseTx(); - void ClearDistributedEraseTx() { DistributedEraseTx = nullptr; } - + const TDistributedEraseTx::TPtr& GetDistributedEraseTx() const { + Y_VERIFY_DEBUG(DistributedEraseTx); + return DistributedEraseTx; + } + bool BuildDistributedEraseTx(); + void ClearDistributedEraseTx() { DistributedEraseTx = nullptr; } + const TCommitWritesTx::TPtr& GetCommitWritesTx() const { Y_VERIFY_DEBUG(CommitWritesTx); return CommitWritesTx; @@ -519,10 +519,10 @@ public: void SetScanTask(ui64 id) { ScanTask = id; } ui64 GetScanTask() const { return ScanTask; } - void SetAsyncJobActor(TActorId aid) { AsyncJobActor = aid; } - TActorId GetAsyncJobActor() const { return AsyncJobActor; } - void KillAsyncJobActor(const TActorContext& ctx); - + void SetAsyncJobActor(TActorId aid) { AsyncJobActor = aid; } + TActorId GetAsyncJobActor() const { return AsyncJobActor; } + void KillAsyncJobActor(const TActorContext& ctx); + void SetStreamSink(TActorId sink) { StreamSink = sink; } TActorId GetStreamSink() const { return StreamSink; } @@ -541,7 +541,7 @@ private: TValidatedDataTx::TPtr DataTx; THolder<NKikimrTxDataShard::TFlatSchemeTransaction> SchemeTx; THolder<NKikimrTxDataShard::TSnapshotTransaction> SnapshotTx; - TDistributedEraseTx::TPtr DistributedEraseTx; + TDistributedEraseTx::TPtr DistributedEraseTx; TCommitWritesTx::TPtr CommitWritesTx; TString TxBody; @@ -552,10 +552,10 @@ private: ui64 SchemeShardId; ui64 SubDomainPathId; NKikimrSubDomains::TProcessingParams ProcessingParams; - TSchemaOperation::EType SchemeTxType; + TSchemaOperation::EType SchemeTxType; ui64 ScanSnapshotId; ui64 ScanTask; - TActorId AsyncJobActor; + TActorId AsyncJobActor; TActorId StreamSink; TActorId ScanActor; ui64 PageFaultCount = 0; diff --git a/ydb/core/tx/datashard/datashard_change_receiving.cpp b/ydb/core/tx/datashard/datashard_change_receiving.cpp index 9a03c939ba7..fcb74f44019 100644 --- a/ydb/core/tx/datashard/datashard_change_receiving.cpp +++ b/ydb/core/tx/datashard/datashard_change_receiving.cpp @@ -1,369 +1,369 @@ -#include "datashard_impl.h" - -namespace NKikimr { +#include "datashard_impl.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + class TDataShard::TTxChangeExchangeHandshake: public TTransactionBase<TDataShard> { using Schema = TDataShard::Schema; - -public: + +public: explicit TTxChangeExchangeHandshake(TDataShard* self, TEvChangeExchange::TEvHandshake::TPtr ev) - : TTransactionBase(self) - , Ev(std::move(ev)) - , Status(new TEvChangeExchange::TEvStatus) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CHANGE_EXCHANGE_HANDSHAKE; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - if (Self->State != TShardState::Ready) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); - return true; - } - - const auto& msg = Ev->Get()->Record; - - auto it = Self->InChangeSenders.find(msg.GetOrigin()); - if (it == Self->InChangeSenders.end()) { - NIceDb::TNiceDb db(txc.DB); - - auto rowset = db.Table<Schema::ChangeSenders>().Key(msg.GetOrigin()).Select(); - if (!rowset.IsReady()) { - return false; - } - - if (rowset.IsValid()) { - const auto generation = rowset.GetValue<Schema::ChangeSenders::Generation>(); - const auto lastRecordOrder = rowset.GetValueOrDefault<Schema::ChangeSenders::LastRecordOrder>(0); - it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(generation, lastRecordOrder)).first; - } else { - it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(msg.GetGeneration())).first; - } - } - - if (it->second.Generation > msg.GetGeneration()) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); - } else { - it->second.Generation = msg.GetGeneration(); - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); - Status->Record.SetLastRecordOrder(it->second.LastRecordOrder); - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::ChangeSenders>().Key(it->first).Update( - NIceDb::TUpdate<Schema::ChangeSenders::Generation>(it->second.Generation), - NIceDb::TUpdate<Schema::ChangeSenders::LastSeenAt>(ctx.Now().GetValue()) - ); - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - Y_VERIFY(Status); - ctx.Send(Ev->Sender, Status.Release()); - } - -private: - TEvChangeExchange::TEvHandshake::TPtr Ev; - THolder<TEvChangeExchange::TEvStatus> Status; - -}; // TTxChangeExchangeHandshake - + : TTransactionBase(self) + , Ev(std::move(ev)) + , Status(new TEvChangeExchange::TEvStatus) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CHANGE_EXCHANGE_HANDSHAKE; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + if (Self->State != TShardState::Ready) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); + return true; + } + + const auto& msg = Ev->Get()->Record; + + auto it = Self->InChangeSenders.find(msg.GetOrigin()); + if (it == Self->InChangeSenders.end()) { + NIceDb::TNiceDb db(txc.DB); + + auto rowset = db.Table<Schema::ChangeSenders>().Key(msg.GetOrigin()).Select(); + if (!rowset.IsReady()) { + return false; + } + + if (rowset.IsValid()) { + const auto generation = rowset.GetValue<Schema::ChangeSenders::Generation>(); + const auto lastRecordOrder = rowset.GetValueOrDefault<Schema::ChangeSenders::LastRecordOrder>(0); + it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(generation, lastRecordOrder)).first; + } else { + it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(msg.GetGeneration())).first; + } + } + + if (it->second.Generation > msg.GetGeneration()) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); + } else { + it->second.Generation = msg.GetGeneration(); + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); + Status->Record.SetLastRecordOrder(it->second.LastRecordOrder); + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::ChangeSenders>().Key(it->first).Update( + NIceDb::TUpdate<Schema::ChangeSenders::Generation>(it->second.Generation), + NIceDb::TUpdate<Schema::ChangeSenders::LastSeenAt>(ctx.Now().GetValue()) + ); + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + Y_VERIFY(Status); + ctx.Send(Ev->Sender, Status.Release()); + } + +private: + TEvChangeExchange::TEvHandshake::TPtr Ev; + THolder<TEvChangeExchange::TEvStatus> Status; + +}; // TTxChangeExchangeHandshake + class TDataShard::TTxApplyChangeRecords: public TTransactionBase<TDataShard> { static NTable::ERowOp GetRowOperation(const NKikimrChangeExchange::TChangeRecord::TDataChange& record) { - switch (record.GetRowOperationCase()) { - case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + switch (record.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: return NTable::ERowOp::Upsert; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: + case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: return NTable::ERowOp::Erase; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: + case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: return NTable::ERowOp::Reset; - default: + default: return NTable::ERowOp::Absent; - } - } - - static auto& GetValue(const NKikimrChangeExchange::TChangeRecord::TDataChange& record) { - switch (record.GetRowOperationCase()) { - case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: - return record.GetUpsert(); - case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: - return record.GetReset(); - default: - Y_FAIL_S("Unexpected row operation: " << static_cast<ui32>(record.GetRowOperationCase())); - } - } - - void AddRecordStatus(const TActorContext& ctx, ui64 order, NKikimrChangeExchange::TEvStatus::EStatus status, - NKikimrChangeExchange::TEvStatus::EReason reason = NKikimrChangeExchange::TEvStatus::REASON_NONE, - const TString& error = {}) - { - auto& recordStatus = *Status->Record.AddRecordStatuses(); - recordStatus.SetOrder(order); - recordStatus.SetStatus(status); - recordStatus.SetReason(reason); - - if (error) { - LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot apply change record" - << ": error# " << error - << ", at tablet# " << Self->TabletID()); - } - - if (status == NKikimrChangeExchange::TEvStatus::STATUS_REJECT) { - Status->Record.SetStatus(status); - } - - if (Status->Record.GetStatus() != NKikimrChangeExchange::TEvStatus::STATUS_REJECT) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); - Status->Record.SetLastRecordOrder(order); - } - } - - bool ProcessRecord(const NKikimrChangeExchange::TChangeRecord& record, TTransactionContext& txc, const TActorContext& ctx) { - Key.clear(); - Value.clear(); - - if (record.GetKindCase() != NKikimrChangeExchange::TChangeRecord::kAsyncIndex) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_UNEXPECTED_KIND, - TStringBuilder() << "Unexpected kind: " << static_cast<ui32>(record.GetKindCase())); - return false; - } - - const auto& userTables = Self->GetUserTables(); - auto it = userTables.find(record.GetLocalPathId()); - if (it == userTables.end()) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Unknown table with id: " << record.GetLocalPathId()); - return false; - } - - const auto& tableInfo = *it->second; - const auto& asyncIndex = record.GetAsyncIndex(); - const auto& serializedKey = asyncIndex.GetKey(); - - if (serializedKey.TagsSize() != tableInfo.KeyColumnIds.size()) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Key column count mismatch" - << ": got " << serializedKey.TagsSize() - << ", expected " << tableInfo.KeyColumnIds.size()); - return false; - } - - for (size_t i = 0; i < tableInfo.KeyColumnIds.size(); ++i) { - if (serializedKey.GetTags(i) != tableInfo.KeyColumnIds.at(i)) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Key column schema mismatch at position: " << i); - return false; - } - } - - if (!TSerializedCellVec::TryParse(serializedKey.GetData(), KeyCells)) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, "Cannot parse key"); - return false; - } - - if (KeyCells.GetCells().size() != tableInfo.KeyColumnTypes.size()) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Cell count doesn't match row scheme" - << ": got " << KeyCells.GetCells().size() - << ", expected " << tableInfo.KeyColumnTypes.size()); - return false; - } - - ui64 keyBytes = 0; - for (size_t i = 0; i < tableInfo.KeyColumnTypes.size(); ++i) { - const auto type = tableInfo.KeyColumnTypes.at(i); - const auto& cell = KeyCells.GetCells().at(i); - - if (type == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - "Keys with Uint8 column values >127 are currently prohibited"); - return false; - } - - keyBytes += cell.Size(); - Key.emplace_back(cell.AsRef(), type); - } - - if (keyBytes > NLimits::MaxWriteKeySize) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Key is too big" - << ": actual " << keyBytes << " bytes" - << ", limit " << NLimits::MaxWriteKeySize << " bytes"); - return false; - } - + } + } + + static auto& GetValue(const NKikimrChangeExchange::TChangeRecord::TDataChange& record) { + switch (record.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + return record.GetUpsert(); + case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: + return record.GetReset(); + default: + Y_FAIL_S("Unexpected row operation: " << static_cast<ui32>(record.GetRowOperationCase())); + } + } + + void AddRecordStatus(const TActorContext& ctx, ui64 order, NKikimrChangeExchange::TEvStatus::EStatus status, + NKikimrChangeExchange::TEvStatus::EReason reason = NKikimrChangeExchange::TEvStatus::REASON_NONE, + const TString& error = {}) + { + auto& recordStatus = *Status->Record.AddRecordStatuses(); + recordStatus.SetOrder(order); + recordStatus.SetStatus(status); + recordStatus.SetReason(reason); + + if (error) { + LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, "Cannot apply change record" + << ": error# " << error + << ", at tablet# " << Self->TabletID()); + } + + if (status == NKikimrChangeExchange::TEvStatus::STATUS_REJECT) { + Status->Record.SetStatus(status); + } + + if (Status->Record.GetStatus() != NKikimrChangeExchange::TEvStatus::STATUS_REJECT) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); + Status->Record.SetLastRecordOrder(order); + } + } + + bool ProcessRecord(const NKikimrChangeExchange::TChangeRecord& record, TTransactionContext& txc, const TActorContext& ctx) { + Key.clear(); + Value.clear(); + + if (record.GetKindCase() != NKikimrChangeExchange::TChangeRecord::kAsyncIndex) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_UNEXPECTED_KIND, + TStringBuilder() << "Unexpected kind: " << static_cast<ui32>(record.GetKindCase())); + return false; + } + + const auto& userTables = Self->GetUserTables(); + auto it = userTables.find(record.GetLocalPathId()); + if (it == userTables.end()) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Unknown table with id: " << record.GetLocalPathId()); + return false; + } + + const auto& tableInfo = *it->second; + const auto& asyncIndex = record.GetAsyncIndex(); + const auto& serializedKey = asyncIndex.GetKey(); + + if (serializedKey.TagsSize() != tableInfo.KeyColumnIds.size()) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Key column count mismatch" + << ": got " << serializedKey.TagsSize() + << ", expected " << tableInfo.KeyColumnIds.size()); + return false; + } + + for (size_t i = 0; i < tableInfo.KeyColumnIds.size(); ++i) { + if (serializedKey.GetTags(i) != tableInfo.KeyColumnIds.at(i)) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Key column schema mismatch at position: " << i); + return false; + } + } + + if (!TSerializedCellVec::TryParse(serializedKey.GetData(), KeyCells)) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, "Cannot parse key"); + return false; + } + + if (KeyCells.GetCells().size() != tableInfo.KeyColumnTypes.size()) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Cell count doesn't match row scheme" + << ": got " << KeyCells.GetCells().size() + << ", expected " << tableInfo.KeyColumnTypes.size()); + return false; + } + + ui64 keyBytes = 0; + for (size_t i = 0; i < tableInfo.KeyColumnTypes.size(); ++i) { + const auto type = tableInfo.KeyColumnTypes.at(i); + const auto& cell = KeyCells.GetCells().at(i); + + if (type == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + "Keys with Uint8 column values >127 are currently prohibited"); + return false; + } + + keyBytes += cell.Size(); + Key.emplace_back(cell.AsRef(), type); + } + + if (keyBytes > NLimits::MaxWriteKeySize) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Key is too big" + << ": actual " << keyBytes << " bytes" + << ", limit " << NLimits::MaxWriteKeySize << " bytes"); + return false; + } + const NTable::ERowOp rop = GetRowOperation(asyncIndex); - switch (rop) { + switch (rop) { case NTable::ERowOp::Upsert: case NTable::ERowOp::Reset: { - const auto& serializedValue = GetValue(asyncIndex); - - if (!TSerializedCellVec::TryParse(serializedValue.GetData(), ValueCells)) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, "Cannot parse value"); - return false; - } - - if (serializedValue.TagsSize() != ValueCells.GetCells().size()) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Cell count doesn't match row scheme" - << ": got " << ValueCells.GetCells().size() - << ", expected " << serializedValue.TagsSize()); - return false; - } - - for (size_t i = 0; i < ValueCells.GetCells().size(); ++i) { - const auto tag = serializedValue.GetTags(i); - - const auto* column = tableInfo.Columns.FindPtr(tag); - if (!column) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Missing column with id: " << tag); - return false; - } - - const auto& cell = ValueCells.GetCells().at(i); - if (cell.Size() > NLimits::MaxWriteValueSize) { - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, - TStringBuilder() << "Value cell is too big" - << ": actual " << cell.Size() << " bytes" - << ", limit " << NLimits::MaxWriteValueSize << " bytes"); - return false; - } - + const auto& serializedValue = GetValue(asyncIndex); + + if (!TSerializedCellVec::TryParse(serializedValue.GetData(), ValueCells)) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, "Cannot parse value"); + return false; + } + + if (serializedValue.TagsSize() != ValueCells.GetCells().size()) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Cell count doesn't match row scheme" + << ": got " << ValueCells.GetCells().size() + << ", expected " << serializedValue.TagsSize()); + return false; + } + + for (size_t i = 0; i < ValueCells.GetCells().size(); ++i) { + const auto tag = serializedValue.GetTags(i); + + const auto* column = tableInfo.Columns.FindPtr(tag); + if (!column) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Missing column with id: " << tag); + return false; + } + + const auto& cell = ValueCells.GetCells().at(i); + if (cell.Size() > NLimits::MaxWriteValueSize) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, + TStringBuilder() << "Value cell is too big" + << ": actual " << cell.Size() << " bytes" + << ", limit " << NLimits::MaxWriteValueSize << " bytes"); + return false; + } + Value.emplace_back(tag, NTable::ECellOp::Set, TRawTypeValue(cell.AsRef(), column->Type)); - } - - break; - } - + } + + break; + } + case NTable::ERowOp::Erase: - break; - + break; + case NTable::ERowOp::Absent: - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, - NKikimrChangeExchange::TEvStatus::REASON_UNEXPECTED_ROW_OPERATION, "Row operation is absent"); - return false; - } - - txc.DB.Update(tableInfo.LocalTid, rop, Key, Value, TRowVersion(record.GetStep(), record.GetTxId())); - AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_OK); - - return true; - } - -public: + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_UNEXPECTED_ROW_OPERATION, "Row operation is absent"); + return false; + } + + txc.DB.Update(tableInfo.LocalTid, rop, Key, Value, TRowVersion(record.GetStep(), record.GetTxId())); + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_OK); + + return true; + } + +public: explicit TTxApplyChangeRecords(TDataShard* self, TEvChangeExchange::TEvApplyRecords::TPtr ev) - : TTransactionBase(self) - , Ev(std::move(ev)) - , Status(new TEvChangeExchange::TEvStatus) - { - } - - TTxType GetTxType() const override { - return TXTYPE_APPLY_CHANGE_RECORDS; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - if (Self->State != TShardState::Ready) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); - return true; - } - - const auto& msg = Ev->Get()->Record; - - auto it = Self->InChangeSenders.find(msg.GetOrigin()); - if (it == Self->InChangeSenders.end()) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_UNKNOWN_ORIGIN); - return true; - } else if (it->second.Generation > msg.GetGeneration()) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); - return true; - } else if (it->second.Generation != msg.GetGeneration()) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_UNKNOWN_ORIGIN); - return true; - } - - auto completeEdge = TRowVersion::Min(); - for (const auto& record : msg.GetRecords()) { - if (ProcessRecord(record, txc, ctx)) { - completeEdge = Max(completeEdge, TRowVersion(record.GetStep(), record.GetTxId())); - } else { - break; - } - } - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::ChangeSenders>().Key(it->first).Update( - NIceDb::TUpdate<Schema::ChangeSenders::Generation>(it->second.Generation), - NIceDb::TUpdate<Schema::ChangeSenders::LastSeenAt>(ctx.Now().GetValue()) - ); - - if (Status->Record.HasLastRecordOrder()) { - it->second.LastRecordOrder = Status->Record.GetLastRecordOrder(); - db.Table<Schema::ChangeSenders>().Key(it->first).Update( - NIceDb::TUpdate<Schema::ChangeSenders::LastRecordOrder>(it->second.LastRecordOrder) - ); - } - - if (completeEdge) { - Self->PromoteCompleteEdge(completeEdge, txc); - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - Y_VERIFY(Status); - ctx.Send(Ev->Sender, Status.Release()); - } - -private: - TEvChangeExchange::TEvApplyRecords::TPtr Ev; - THolder<TEvChangeExchange::TEvStatus> Status; - - TSerializedCellVec KeyCells; - TSerializedCellVec ValueCells; - - TVector<TRawTypeValue> Key; - TVector<NTable::TUpdateOp> Value; - -}; // TTxApplyChangeRecords - + : TTransactionBase(self) + , Ev(std::move(ev)) + , Status(new TEvChangeExchange::TEvStatus) + { + } + + TTxType GetTxType() const override { + return TXTYPE_APPLY_CHANGE_RECORDS; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + if (Self->State != TShardState::Ready) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); + return true; + } + + const auto& msg = Ev->Get()->Record; + + auto it = Self->InChangeSenders.find(msg.GetOrigin()); + if (it == Self->InChangeSenders.end()) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_UNKNOWN_ORIGIN); + return true; + } else if (it->second.Generation > msg.GetGeneration()) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); + return true; + } else if (it->second.Generation != msg.GetGeneration()) { + Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_UNKNOWN_ORIGIN); + return true; + } + + auto completeEdge = TRowVersion::Min(); + for (const auto& record : msg.GetRecords()) { + if (ProcessRecord(record, txc, ctx)) { + completeEdge = Max(completeEdge, TRowVersion(record.GetStep(), record.GetTxId())); + } else { + break; + } + } + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::ChangeSenders>().Key(it->first).Update( + NIceDb::TUpdate<Schema::ChangeSenders::Generation>(it->second.Generation), + NIceDb::TUpdate<Schema::ChangeSenders::LastSeenAt>(ctx.Now().GetValue()) + ); + + if (Status->Record.HasLastRecordOrder()) { + it->second.LastRecordOrder = Status->Record.GetLastRecordOrder(); + db.Table<Schema::ChangeSenders>().Key(it->first).Update( + NIceDb::TUpdate<Schema::ChangeSenders::LastRecordOrder>(it->second.LastRecordOrder) + ); + } + + if (completeEdge) { + Self->PromoteCompleteEdge(completeEdge, txc); + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + Y_VERIFY(Status); + ctx.Send(Ev->Sender, Status.Release()); + } + +private: + TEvChangeExchange::TEvApplyRecords::TPtr Ev; + THolder<TEvChangeExchange::TEvStatus> Status; + + TSerializedCellVec KeyCells; + TSerializedCellVec ValueCells; + + TVector<TRawTypeValue> Key; + TVector<NTable::TUpdateOp> Value; + +}; // TTxApplyChangeRecords + void TDataShard::Handle(TEvChangeExchange::TEvHandshake::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxChangeExchangeHandshake(this, ev), ctx); -} - + Execute(new TTxChangeExchangeHandshake(this, ev), ctx); +} + void TDataShard::Handle(TEvChangeExchange::TEvApplyRecords::TPtr& ev, const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Handle TEvChangeExchange::TEvApplyRecords" << ": origin# " << ev->Get()->Record.GetOrigin() << ", generation# " << ev->Get()->Record.GetGeneration() << ", at tablet# " << TabletID()); - Execute(new TTxApplyChangeRecords(this, ev), ctx); -} - + Execute(new TTxApplyChangeRecords(this, ev), ctx); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_change_sender_activation.cpp b/ydb/core/tx/datashard/datashard_change_sender_activation.cpp index 59a26357882..2635668bece 100644 --- a/ydb/core/tx/datashard/datashard_change_sender_activation.cpp +++ b/ydb/core/tx/datashard/datashard_change_sender_activation.cpp @@ -1,130 +1,130 @@ -#include "datashard_impl.h" - -namespace NKikimr { +#include "datashard_impl.h" + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + class TDataShard::TTxActivateChangeSender: public TTransactionBase<TDataShard> { -public: +public: explicit TTxActivateChangeSender(TDataShard* self, ui64 origin, const TActorId& ackTo) - : TTransactionBase(self) - , Origin(origin) - , AckTo(ackTo) - , AllSrcActivationsReceived(false) - { - } - - TTxType GetTxType() const override { - return TXTYPE_ACTIVATE_CHANGE_SENDER; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSender Execute" - << ": origin# " << Origin - << ", at tablet# " << Self->TabletID()); - - if (!Self->ReceiveActivationsFrom.contains(Origin)) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Ignoring received activation" - << ": origin# " << Origin - << ", at tablet# " << Self->TabletID()); - return true; - } - - Self->ReceiveActivationsFrom.erase(Origin); - AllSrcActivationsReceived = !Self->ReceiveActivationsFrom; - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::DstChangeSenderActivations>().Key(Origin).Delete(); - - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSender Complete" - << ": origin# " << Origin - << ", at tablet# " << Self->TabletID()); - - auto ev = MakeHolder<TEvChangeExchange::TEvActivateSenderAck>(); - ev->Record.SetOrigin(Self->TabletID()); - ctx.Send(AckTo, ev.Release()); - - if (AllSrcActivationsReceived) { - if (!Self->GetChangeSender()) { - // There might be a race between TxInit::Complete and <this>::Complete - // so just skip it. Change sender will be activated upon TxInit::Complete. - return; - } - - Self->MaybeActivateChangeSender(ctx); - } - } - -private: - const ui64 Origin; - const TActorId AckTo; - bool AllSrcActivationsReceived; - -}; // TTxActivateChangeSender - + : TTransactionBase(self) + , Origin(origin) + , AckTo(ackTo) + , AllSrcActivationsReceived(false) + { + } + + TTxType GetTxType() const override { + return TXTYPE_ACTIVATE_CHANGE_SENDER; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSender Execute" + << ": origin# " << Origin + << ", at tablet# " << Self->TabletID()); + + if (!Self->ReceiveActivationsFrom.contains(Origin)) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Ignoring received activation" + << ": origin# " << Origin + << ", at tablet# " << Self->TabletID()); + return true; + } + + Self->ReceiveActivationsFrom.erase(Origin); + AllSrcActivationsReceived = !Self->ReceiveActivationsFrom; + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::DstChangeSenderActivations>().Key(Origin).Delete(); + + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSender Complete" + << ": origin# " << Origin + << ", at tablet# " << Self->TabletID()); + + auto ev = MakeHolder<TEvChangeExchange::TEvActivateSenderAck>(); + ev->Record.SetOrigin(Self->TabletID()); + ctx.Send(AckTo, ev.Release()); + + if (AllSrcActivationsReceived) { + if (!Self->GetChangeSender()) { + // There might be a race between TxInit::Complete and <this>::Complete + // so just skip it. Change sender will be activated upon TxInit::Complete. + return; + } + + Self->MaybeActivateChangeSender(ctx); + } + } + +private: + const ui64 Origin; + const TActorId AckTo; + bool AllSrcActivationsReceived; + +}; // TTxActivateChangeSender + class TDataShard::TTxActivateChangeSenderAck: public TTransactionBase<TDataShard> { -public: +public: explicit TTxActivateChangeSenderAck(TDataShard* self, ui64 origin) - : TTransactionBase(self) - , Origin(origin) - , AllDstAcksReceived(false) - { - } - - TTxType GetTxType() const override { - return TXTYPE_ACTIVATE_CHANGE_SENDER_ACK; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSenderAck Execute" - << ": origin# " << Origin - << ", at tablet# " << Self->TabletID()); - - Self->ChangeSenderActivator.Ack(Origin, ctx); - AllDstAcksReceived = Self->ChangeSenderActivator.AllAcked(); - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::SrcChangeSenderActivations>().Key(Origin).Delete(); - - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSenderAck Complete" - << ": origin# " << Origin - << ", at tablet# " << Self->TabletID()); - - if (AllDstAcksReceived) { - for (const auto& [ackTo, opIds] : Self->SrcAckPartitioningChangedTo) { - for (const ui64 opId : opIds) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " ack split partitioning changed to schemeshard " << opId); - ctx.Send(ackTo, new TEvDataShard::TEvSplitPartitioningChangedAck(opId, Self->TabletID())); - } - } - - Self->SrcAckPartitioningChangedTo.clear(); - Self->CheckStateChange(ctx); - } - } - -private: - const ui64 Origin; - bool AllDstAcksReceived; - -}; // TTxActivateChangeSenderAck - + : TTransactionBase(self) + , Origin(origin) + , AllDstAcksReceived(false) + { + } + + TTxType GetTxType() const override { + return TXTYPE_ACTIVATE_CHANGE_SENDER_ACK; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSenderAck Execute" + << ": origin# " << Origin + << ", at tablet# " << Self->TabletID()); + + Self->ChangeSenderActivator.Ack(Origin, ctx); + AllDstAcksReceived = Self->ChangeSenderActivator.AllAcked(); + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::SrcChangeSenderActivations>().Key(Origin).Delete(); + + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxActivateChangeSenderAck Complete" + << ": origin# " << Origin + << ", at tablet# " << Self->TabletID()); + + if (AllDstAcksReceived) { + for (const auto& [ackTo, opIds] : Self->SrcAckPartitioningChangedTo) { + for (const ui64 opId : opIds) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " ack split partitioning changed to schemeshard " << opId); + ctx.Send(ackTo, new TEvDataShard::TEvSplitPartitioningChangedAck(opId, Self->TabletID())); + } + } + + Self->SrcAckPartitioningChangedTo.clear(); + Self->CheckStateChange(ctx); + } + } + +private: + const ui64 Origin; + bool AllDstAcksReceived; + +}; // TTxActivateChangeSenderAck + void TDataShard::Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxActivateChangeSender(this, ev->Get()->Record.GetOrigin(), ev->Sender), ctx); -} - + Execute(new TTxActivateChangeSender(this, ev->Get()->Record.GetOrigin(), ev->Sender), ctx); +} + void TDataShard::Handle(TEvChangeExchange::TEvActivateSenderAck::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxActivateChangeSenderAck(this, ev->Get()->Record.GetOrigin()), ctx); -} - + Execute(new TTxActivateChangeSenderAck(this, ev->Get()->Record.GetOrigin()), ctx); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_change_sending.cpp b/ydb/core/tx/datashard/datashard_change_sending.cpp index d0f12fcc067..7789c5137e8 100644 --- a/ydb/core/tx/datashard/datashard_change_sending.cpp +++ b/ydb/core/tx/datashard/datashard_change_sending.cpp @@ -1,327 +1,327 @@ -#include "datashard_impl.h" - -#include <util/generic/size_literals.h> - -namespace NKikimr { +#include "datashard_impl.h" + +#include <util/generic/size_literals.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + class TDataShard::TTxRequestChangeRecords: public TTransactionBase<TDataShard> { using Schema = TDataShard::Schema; - - bool Precharge(NIceDb::TNiceDb& db) { - size_t bodiesSize = 0; - bool ok = true; - - for (const auto& [_, records] : Self->ChangeRecordsRequested) { - for (const auto& record : records) { - if (bodiesSize && (bodiesSize + record.BodySize) > MemLimit) { - break; - } - - bodiesSize += record.BodySize; - - ok = ok && db.Table<Schema::ChangeRecords>().Key(record.Order).Precharge(); - ok = ok && db.Table<Schema::ChangeRecordDetails>().Key(record.Order).Precharge(); - } - } - - return ok; - } - - bool Select(NIceDb::TNiceDb& db) { - for (auto& [recipient, records] : Self->ChangeRecordsRequested) { - if (!records) { - continue; - } - - auto it = records.begin(); - while (it != records.end()) { - if (MemUsage && (MemUsage + it->BodySize) > MemLimit) { - break; - } - - auto basic = db.Table<Schema::ChangeRecords>().Key(it->Order).Select(); - auto details = db.Table<Schema::ChangeRecordDetails>().Key(it->Order).Select(); - - if (!basic.IsReady() || !details.IsReady()) { - return false; - } - - if (!basic.IsValid() && !details.IsValid()) { - RecordsToForget[recipient].emplace_back(it->Order); - it = records.erase(it); - continue; - } - - Y_VERIFY_S(basic.IsValid() && details.IsValid(), "Inconsistent basic and details" - << ", basic.IsValid: " << basic.IsValid() - << ", details.IsValid: " << details.IsValid() - << ", recipient: " << recipient - << ", records.size: " << records.size() - << ", it->Order: " << it->Order - << ", it->BodySize: " << it->BodySize); - - RecordsToSend[recipient].emplace_back(TChangeRecordBuilder(details.GetValue<Schema::ChangeRecordDetails::Kind>()) - .WithOrder(it->Order) - .WithGroup(basic.GetValue<Schema::ChangeRecords::Group>()) - .WithStep(basic.GetValue<Schema::ChangeRecords::PlanStep>()) - .WithTxId(basic.GetValue<Schema::ChangeRecords::TxId>()) - .WithPathId(TPathId( - basic.GetValue<Schema::ChangeRecords::PathOwnerId>(), - basic.GetValue<Schema::ChangeRecords::LocalPathId>() - )) - .WithBody(details.GetValue<Schema::ChangeRecordDetails::Body>()) - .Build()); - - MemUsage += it->BodySize; - it = records.erase(it); - } - } - - return true; - } - -public: + + bool Precharge(NIceDb::TNiceDb& db) { + size_t bodiesSize = 0; + bool ok = true; + + for (const auto& [_, records] : Self->ChangeRecordsRequested) { + for (const auto& record : records) { + if (bodiesSize && (bodiesSize + record.BodySize) > MemLimit) { + break; + } + + bodiesSize += record.BodySize; + + ok = ok && db.Table<Schema::ChangeRecords>().Key(record.Order).Precharge(); + ok = ok && db.Table<Schema::ChangeRecordDetails>().Key(record.Order).Precharge(); + } + } + + return ok; + } + + bool Select(NIceDb::TNiceDb& db) { + for (auto& [recipient, records] : Self->ChangeRecordsRequested) { + if (!records) { + continue; + } + + auto it = records.begin(); + while (it != records.end()) { + if (MemUsage && (MemUsage + it->BodySize) > MemLimit) { + break; + } + + auto basic = db.Table<Schema::ChangeRecords>().Key(it->Order).Select(); + auto details = db.Table<Schema::ChangeRecordDetails>().Key(it->Order).Select(); + + if (!basic.IsReady() || !details.IsReady()) { + return false; + } + + if (!basic.IsValid() && !details.IsValid()) { + RecordsToForget[recipient].emplace_back(it->Order); + it = records.erase(it); + continue; + } + + Y_VERIFY_S(basic.IsValid() && details.IsValid(), "Inconsistent basic and details" + << ", basic.IsValid: " << basic.IsValid() + << ", details.IsValid: " << details.IsValid() + << ", recipient: " << recipient + << ", records.size: " << records.size() + << ", it->Order: " << it->Order + << ", it->BodySize: " << it->BodySize); + + RecordsToSend[recipient].emplace_back(TChangeRecordBuilder(details.GetValue<Schema::ChangeRecordDetails::Kind>()) + .WithOrder(it->Order) + .WithGroup(basic.GetValue<Schema::ChangeRecords::Group>()) + .WithStep(basic.GetValue<Schema::ChangeRecords::PlanStep>()) + .WithTxId(basic.GetValue<Schema::ChangeRecords::TxId>()) + .WithPathId(TPathId( + basic.GetValue<Schema::ChangeRecords::PathOwnerId>(), + basic.GetValue<Schema::ChangeRecords::LocalPathId>() + )) + .WithBody(details.GetValue<Schema::ChangeRecordDetails::Body>()) + .Build()); + + MemUsage += it->BodySize; + it = records.erase(it); + } + } + + return true; + } + +public: explicit TTxRequestChangeRecords(TDataShard* self) - : TTransactionBase(self) - { - } - - TTxType GetTxType() const override { - return TXTYPE_REQUEST_CHANGE_RECORDS; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRequestChangeRecords Execute" - << ": at tablet# " << Self->TabletID()); - - NIceDb::TNiceDb db(txc.DB); - if (!Precharge(db) || !Select(db)) { - return false; - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - size_t sent = 0; - for (auto& kv : RecordsToSend) { - const auto& to = kv.first; - auto& records = kv.second; - - sent += records.size(); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Send " << records.size() << " change records" - << ": to# " << to - << ", at tablet# " << Self->TabletID()); - ctx.Send(to, new TEvChangeExchange::TEvRecords(std::move(records))); - } - - size_t forgotten = 0; - for (auto& kv : RecordsToForget) { - const auto& to = kv.first; - auto& records = kv.second; - - forgotten += records.size(); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Forget " << records.size() << " change records" - << ": to# " << to - << ", at tablet# " << Self->TabletID()); - ctx.Send(to, new TEvChangeExchange::TEvForgetRecords(std::move(records))); - } - - size_t left = 0; - for (const auto& [_, records] : Self->ChangeRecordsRequested) { - left += records.size(); - } - - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRequestChangeRecords Complete" - << ": sent# " << sent - << ", forgotten# " << forgotten - << ", left# " << left - << ", at tablet# " << Self->TabletID()); - - if (left) { - Self->Execute(new TTxRequestChangeRecords(Self), ctx); - } else { - Self->RequestChangeRecordsInFly = false; - } - } - -private: - static constexpr size_t MemLimit = 512_KB; - size_t MemUsage = 0; - - THashMap<TActorId, TVector<TChangeRecord>> RecordsToSend; - THashMap<TActorId, TVector<ui64>> RecordsToForget; - -}; // TTxRequestChangeRecords - + : TTransactionBase(self) + { + } + + TTxType GetTxType() const override { + return TXTYPE_REQUEST_CHANGE_RECORDS; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRequestChangeRecords Execute" + << ": at tablet# " << Self->TabletID()); + + NIceDb::TNiceDb db(txc.DB); + if (!Precharge(db) || !Select(db)) { + return false; + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + size_t sent = 0; + for (auto& kv : RecordsToSend) { + const auto& to = kv.first; + auto& records = kv.second; + + sent += records.size(); + + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Send " << records.size() << " change records" + << ": to# " << to + << ", at tablet# " << Self->TabletID()); + ctx.Send(to, new TEvChangeExchange::TEvRecords(std::move(records))); + } + + size_t forgotten = 0; + for (auto& kv : RecordsToForget) { + const auto& to = kv.first; + auto& records = kv.second; + + forgotten += records.size(); + + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Forget " << records.size() << " change records" + << ": to# " << to + << ", at tablet# " << Self->TabletID()); + ctx.Send(to, new TEvChangeExchange::TEvForgetRecords(std::move(records))); + } + + size_t left = 0; + for (const auto& [_, records] : Self->ChangeRecordsRequested) { + left += records.size(); + } + + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRequestChangeRecords Complete" + << ": sent# " << sent + << ", forgotten# " << forgotten + << ", left# " << left + << ", at tablet# " << Self->TabletID()); + + if (left) { + Self->Execute(new TTxRequestChangeRecords(Self), ctx); + } else { + Self->RequestChangeRecordsInFly = false; + } + } + +private: + static constexpr size_t MemLimit = 512_KB; + size_t MemUsage = 0; + + THashMap<TActorId, TVector<TChangeRecord>> RecordsToSend; + THashMap<TActorId, TVector<ui64>> RecordsToForget; + +}; // TTxRequestChangeRecords + class TDataShard::TTxRemoveChangeRecords: public TTransactionBase<TDataShard> { - void FillActivationList() { - if (!Self->ChangesQueue) { - if (!Self->ChangeExchangeSplitter.Done()) { - ChangeExchangeSplit = true; - } else { - for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { - if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { - ActivationList.insert(dstTabletId); - } - } - } - } - } - -public: + void FillActivationList() { + if (!Self->ChangesQueue) { + if (!Self->ChangeExchangeSplitter.Done()) { + ChangeExchangeSplit = true; + } else { + for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { + if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { + ActivationList.insert(dstTabletId); + } + } + } + } + } + +public: explicit TTxRemoveChangeRecords(TDataShard* self) - : TTransactionBase(self) - { - } - - TTxType GetTxType() const override { - return TXTYPE_REMOVE_CHANGE_RECORDS; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRemoveChangeRecords Execute" - << ": records# " << Self->ChangeRecordsToRemove.size() - << ", at tablet# " << Self->TabletID()); - - if (!Self->ChangeRecordsToRemove) { - FillActivationList(); - return true; - } - - NIceDb::TNiceDb db(txc.DB); - - auto it = Self->ChangeRecordsToRemove.begin(); - while (RemovedCount < BucketSize && it != Self->ChangeRecordsToRemove.end()) { - Self->RemoveChangeRecord(db, *it); - - it = Self->ChangeRecordsToRemove.erase(it); - ++RemovedCount; - } - - FillActivationList(); - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRemoveChangeRecords Complete" - << ": removed# " << RemovedCount - << ", left# " << Self->ChangeRecordsToRemove.size() - << ", at tablet# " << Self->TabletID()); - - if (Self->ChangeRecordsToRemove) { - Self->Execute(new TTxRemoveChangeRecords(Self), ctx); - } else { - Self->RemoveChangeRecordsInFly = false; - } - - if (ChangeExchangeSplit) { - Self->ChangeExchangeSplitter.DoSplit(ctx); - } - - for (const auto dstTabletId : ActivationList) { - if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { - Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); - } - } - } - -private: - static constexpr size_t BucketSize = 1000; - size_t RemovedCount = 0; - THashSet<ui64> ActivationList; - bool ChangeExchangeSplit = false; - -}; // TTxRemoveChangeRecords - -class TDataShard::TTxChangeExchangeSplitAck: public TTransactionBase<TDataShard> { -public: - explicit TTxChangeExchangeSplitAck(TDataShard* self) - : TTransactionBase(self) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CHANGE_EXCHANGE_SPLIT_ACK; - } - - bool Execute(TTransactionContext&, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxChangeExchangeSplitAck Execute" - << ", at tablet# " << Self->TabletID()); - - Y_VERIFY(!Self->ChangesQueue); - - Self->ChangeExchangeSplitter.Ack(); - Y_VERIFY(Self->ChangeExchangeSplitter.Done()); - - for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { - if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { - ActivationList.insert(dstTabletId); - } - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "TTxChangeExchangeSplitAck Complete" - << ", at tablet# " << Self->TabletID()); - - for (const auto dstTabletId : ActivationList) { - if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { - Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); - } - } - } - -private: - THashSet<ui64> ActivationList; - -}; // TTxChangeExchangeSplitAck - -/// Request + : TTransactionBase(self) + { + } + + TTxType GetTxType() const override { + return TXTYPE_REMOVE_CHANGE_RECORDS; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRemoveChangeRecords Execute" + << ": records# " << Self->ChangeRecordsToRemove.size() + << ", at tablet# " << Self->TabletID()); + + if (!Self->ChangeRecordsToRemove) { + FillActivationList(); + return true; + } + + NIceDb::TNiceDb db(txc.DB); + + auto it = Self->ChangeRecordsToRemove.begin(); + while (RemovedCount < BucketSize && it != Self->ChangeRecordsToRemove.end()) { + Self->RemoveChangeRecord(db, *it); + + it = Self->ChangeRecordsToRemove.erase(it); + ++RemovedCount; + } + + FillActivationList(); + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "TTxRemoveChangeRecords Complete" + << ": removed# " << RemovedCount + << ", left# " << Self->ChangeRecordsToRemove.size() + << ", at tablet# " << Self->TabletID()); + + if (Self->ChangeRecordsToRemove) { + Self->Execute(new TTxRemoveChangeRecords(Self), ctx); + } else { + Self->RemoveChangeRecordsInFly = false; + } + + if (ChangeExchangeSplit) { + Self->ChangeExchangeSplitter.DoSplit(ctx); + } + + for (const auto dstTabletId : ActivationList) { + if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { + Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); + } + } + } + +private: + static constexpr size_t BucketSize = 1000; + size_t RemovedCount = 0; + THashSet<ui64> ActivationList; + bool ChangeExchangeSplit = false; + +}; // TTxRemoveChangeRecords + +class TDataShard::TTxChangeExchangeSplitAck: public TTransactionBase<TDataShard> { +public: + explicit TTxChangeExchangeSplitAck(TDataShard* self) + : TTransactionBase(self) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CHANGE_EXCHANGE_SPLIT_ACK; + } + + bool Execute(TTransactionContext&, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "TTxChangeExchangeSplitAck Execute" + << ", at tablet# " << Self->TabletID()); + + Y_VERIFY(!Self->ChangesQueue); + + Self->ChangeExchangeSplitter.Ack(); + Y_VERIFY(Self->ChangeExchangeSplitter.Done()); + + for (const auto dstTabletId : Self->ChangeSenderActivator.GetDstSet()) { + if (Self->SplitSrcSnapshotSender.Acked(dstTabletId)) { + ActivationList.insert(dstTabletId); + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "TTxChangeExchangeSplitAck Complete" + << ", at tablet# " << Self->TabletID()); + + for (const auto dstTabletId : ActivationList) { + if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { + Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); + } + } + } + +private: + THashSet<ui64> ActivationList; + +}; // TTxChangeExchangeSplitAck + +/// Request void TDataShard::Handle(TEvChangeExchange::TEvRequestRecords::TPtr& ev, const TActorContext& ctx) { - ChangeRecordsRequested[ev->Sender].insert(ev->Get()->Records.begin(), ev->Get()->Records.end()); - ScheduleRequestChangeRecords(ctx); -} - + ChangeRecordsRequested[ev->Sender].insert(ev->Get()->Records.begin(), ev->Get()->Records.end()); + ScheduleRequestChangeRecords(ctx); +} + void TDataShard::ScheduleRequestChangeRecords(const TActorContext& ctx) { - if (ChangeRecordsRequested && !RequestChangeRecordsInFly) { - ctx.Send(SelfId(), new TEvPrivate::TEvRequestChangeRecords); - RequestChangeRecordsInFly = true; - } -} - + if (ChangeRecordsRequested && !RequestChangeRecordsInFly) { + ctx.Send(SelfId(), new TEvPrivate::TEvRequestChangeRecords); + RequestChangeRecordsInFly = true; + } +} + void TDataShard::Handle(TEvPrivate::TEvRequestChangeRecords::TPtr&, const TActorContext& ctx) { - Execute(new TTxRequestChangeRecords(this), ctx); -} - -/// Remove + Execute(new TTxRequestChangeRecords(this), ctx); +} + +/// Remove void TDataShard::Handle(TEvChangeExchange::TEvRemoveRecords::TPtr& ev, const TActorContext& ctx) { - ChangeRecordsToRemove.insert(ev->Get()->Records.begin(), ev->Get()->Records.end()); - ScheduleRemoveChangeRecords(ctx); -} - + ChangeRecordsToRemove.insert(ev->Get()->Records.begin(), ev->Get()->Records.end()); + ScheduleRemoveChangeRecords(ctx); +} + void TDataShard::ScheduleRemoveChangeRecords(const TActorContext& ctx) { - if (ChangeRecordsToRemove && !RemoveChangeRecordsInFly) { - ctx.Send(SelfId(), new TEvPrivate::TEvRemoveChangeRecords); - RemoveChangeRecordsInFly = true; - } -} - + if (ChangeRecordsToRemove && !RemoveChangeRecordsInFly) { + ctx.Send(SelfId(), new TEvPrivate::TEvRemoveChangeRecords); + RemoveChangeRecordsInFly = true; + } +} + void TDataShard::Handle(TEvPrivate::TEvRemoveChangeRecords::TPtr&, const TActorContext& ctx) { - Execute(new TTxRemoveChangeRecords(this), ctx); -} - -/// SplitAck -void TDataShard::Handle(TEvChangeExchange::TEvSplitAck::TPtr&, const TActorContext& ctx) { - Execute(new TTxChangeExchangeSplitAck(this), ctx); -} - + Execute(new TTxRemoveChangeRecords(this), ctx); +} + +/// SplitAck +void TDataShard::Handle(TEvChangeExchange::TEvSplitAck::TPtr&, const TActorContext& ctx) { + Execute(new TTxChangeExchangeSplitAck(this), ctx); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_common_upload.cpp b/ydb/core/tx/datashard/datashard_common_upload.cpp index c5e298cc3ab..088da2b5213 100644 --- a/ydb/core/tx/datashard/datashard_common_upload.cpp +++ b/ydb/core/tx/datashard/datashard_common_upload.cpp @@ -1,32 +1,32 @@ -#include "change_collector.h" -#include "datashard_common_upload.h" +#include "change_collector.h" +#include "datashard_common_upload.h" namespace NKikimr { namespace NDataShard { -template <typename TEvRequest, typename TEvResponse> -TCommonUploadOps<TEvRequest, TEvResponse>::TCommonUploadOps(typename TEvRequest::TPtr& ev, bool breakLocks, bool collectChanges) - : BreakLocks(breakLocks) - , CollectChanges(collectChanges) - , Ev(ev) +template <typename TEvRequest, typename TEvResponse> +TCommonUploadOps<TEvRequest, TEvResponse>::TCommonUploadOps(typename TEvRequest::TPtr& ev, bool breakLocks, bool collectChanges) + : BreakLocks(breakLocks) + , CollectChanges(collectChanges) + , Ev(ev) { } -template <typename TEvRequest, typename TEvResponse> -bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTransactionContext& txc, - const TRowVersion& readVersion, const TRowVersion& writeVersion) -{ - const auto& record = Ev->Get()->Record; - +template <typename TEvRequest, typename TEvResponse> +bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTransactionContext& txc, + const TRowVersion& readVersion, const TRowVersion& writeVersion) +{ + const auto& record = Ev->Get()->Record; + Result = MakeHolder<TEvResponse>(self->TabletID()); - TInstant deadline = TInstant::MilliSeconds(record.GetCancelDeadlineMs()); + TInstant deadline = TInstant::MilliSeconds(record.GetCancelDeadlineMs()); if (deadline && deadline < AppData()->TimeProvider->Now()) { SetError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Deadline exceeded"); return true; } - const ui64 tableId = record.GetTableId(); + const ui64 tableId = record.GetTableId(); const TTableId fullTableId(self->GetPathOwnerId(), tableId); const ui64 localTableId = self->GetLocalTableId(fullTableId); if (localTableId == 0) { @@ -40,36 +40,36 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans Y_VERIFY(tableInfo.ShadowTid == shadowTableId); // Check schemas - if (record.GetRowScheme().KeyColumnIdsSize() != tableInfo.KeyColumnIds.size()) { + if (record.GetRowScheme().KeyColumnIdsSize() != tableInfo.KeyColumnIds.size()) { SetError(NKikimrTxDataShard::TError::SCHEME_ERROR, Sprintf("Key column count mismatch: got %" PRIu64 ", expected %" PRIu64, - record.GetRowScheme().KeyColumnIdsSize(), tableInfo.KeyColumnIds.size())); + record.GetRowScheme().KeyColumnIdsSize(), tableInfo.KeyColumnIds.size())); return true; } for (size_t i = 0; i < tableInfo.KeyColumnIds.size(); ++i) { - if (record.GetRowScheme().GetKeyColumnIds(i) != tableInfo.KeyColumnIds[i]) { + if (record.GetRowScheme().GetKeyColumnIds(i) != tableInfo.KeyColumnIds[i]) { SetError(NKikimrTxDataShard::TError::SCHEME_ERROR, Sprintf("Key column schema at position %" PRISZT, i)); return true; } } - const bool writeToTableShadow = record.GetWriteToTableShadow(); + const bool writeToTableShadow = record.GetWriteToTableShadow(); const bool readForTableShadow = writeToTableShadow && !shadowTableId; const ui32 writeTableId = writeToTableShadow && shadowTableId ? shadowTableId : localTableId; - if (CollectChanges) { - ChangeCollector.Reset(CreateChangeCollector(*self, txc.DB, tableInfo, true)); - ChangeCollector->SetWriteVersion(writeVersion); - if (ChangeCollector->NeedToReadKeys()) { - ChangeCollector->SetReadVersion(readVersion); - } - } - + if (CollectChanges) { + ChangeCollector.Reset(CreateChangeCollector(*self, txc.DB, tableInfo, true)); + ChangeCollector->SetWriteVersion(writeVersion); + if (ChangeCollector->NeedToReadKeys()) { + ChangeCollector->SetReadVersion(readVersion); + } + } + // Prepare (id, Type) vector for value columns TVector<NTable::TTag> tagsForSelect; TVector<std::pair<ui32, NScheme::TTypeId>> valueCols; - for (const auto& colId : record.GetRowScheme().GetValueColumnIds()) { + for (const auto& colId : record.GetRowScheme().GetValueColumnIds()) { if (readForTableShadow) { tagsForSelect.push_back(colId); } @@ -91,7 +91,7 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans NTable::TRowState rowState; ui64 bytes = 0; - for (const auto& r : record.GetRows()) { + for (const auto& r : record.GetRows()) { // TODO: use safe parsing! keyCells.Parse(r.GetKeyColumns()); valueCells.Parse(r.GetValueColumns()); @@ -135,10 +135,10 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans pageFault = true; } - if (pageFault) { - continue; - } - + if (pageFault) { + continue; + } + if (rowState == NTable::ERowOp::Erase || rowState == NTable::ERowOp::Reset) { // Row has been erased in the past, ignore this upsert continue; @@ -172,73 +172,73 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans continue; } - if (!writeToTableShadow) { - if (ChangeCollector) { - Y_VERIFY(CollectChanges); - + if (!writeToTableShadow) { + if (ChangeCollector) { + Y_VERIFY(CollectChanges); + if (!ChangeCollector->Collect(fullTableId, NTable::ERowOp::Upsert, key, value)) { - pageFault = true; - } - - if (pageFault) { - continue; - } - } - - if (BreakLocks) { - self->SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); - } + pageFault = true; + } + + if (pageFault) { + continue; + } + } + + if (BreakLocks) { + self->SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); + } } txc.DB.Update(writeTableId, NTable::ERowOp::Upsert, key, value, writeVersion); } if (pageFault) { - if (ChangeCollector) { - ChangeCollector->Reset(); - } - + if (ChangeCollector) { + ChangeCollector->Reset(); + } + return false; } - self->IncCounter(COUNTER_UPLOAD_ROWS, record.GetRows().size()); + self->IncCounter(COUNTER_UPLOAD_ROWS, record.GetRows().size()); self->IncCounter(COUNTER_UPLOAD_ROWS_BYTES, bytes); tableInfo.Stats.UpdateTime = TAppData::TimeProvider->Now(); return true; } -template <typename TEvRequest, typename TEvResponse> +template <typename TEvRequest, typename TEvResponse> void TCommonUploadOps<TEvRequest, TEvResponse>::SendResult(TDataShard* self, const TActorContext& ctx) { - Y_VERIFY(Result); - + Y_VERIFY(Result); + if (Result->Record.GetStatus() == NKikimrTxDataShard::TError::OK) { self->IncCounter(COUNTER_BULK_UPSERT_SUCCESS); } else { self->IncCounter(COUNTER_BULK_UPSERT_ERROR); } - - ctx.Send(Ev->Sender, std::move(Result)); + + ctx.Send(Ev->Sender, std::move(Result)); +} + +template <typename TEvRequest, typename TEvResponse> +TVector<NMiniKQL::IChangeCollector::TChange> TCommonUploadOps<TEvRequest, TEvResponse>::GetCollectedChanges() const { + if (!ChangeCollector) { + return {}; + } + + auto changes = std::move(ChangeCollector->GetCollected()); + return changes; } -template <typename TEvRequest, typename TEvResponse> -TVector<NMiniKQL::IChangeCollector::TChange> TCommonUploadOps<TEvRequest, TEvResponse>::GetCollectedChanges() const { - if (!ChangeCollector) { - return {}; - } - - auto changes = std::move(ChangeCollector->GetCollected()); - return changes; -} - -template <typename TEvRequest, typename TEvResponse> -void TCommonUploadOps<TEvRequest, TEvResponse>::SetError(ui32 status, const TString& descr) { - Result->Record.SetStatus(status); - Result->Record.SetErrorDescription(descr); -} - -template class TCommonUploadOps<TEvDataShard::TEvUploadRowsRequest, TEvDataShard::TEvUploadRowsResponse>; -template class TCommonUploadOps<TEvDataShard::TEvUnsafeUploadRowsRequest, TEvDataShard::TEvUnsafeUploadRowsResponse>; - +template <typename TEvRequest, typename TEvResponse> +void TCommonUploadOps<TEvRequest, TEvResponse>::SetError(ui32 status, const TString& descr) { + Result->Record.SetStatus(status); + Result->Record.SetErrorDescription(descr); +} + +template class TCommonUploadOps<TEvDataShard::TEvUploadRowsRequest, TEvDataShard::TEvUploadRowsResponse>; +template class TCommonUploadOps<TEvDataShard::TEvUnsafeUploadRowsRequest, TEvDataShard::TEvUnsafeUploadRowsResponse>; + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_common_upload.h b/ydb/core/tx/datashard/datashard_common_upload.h index 3f5a4b36fd9..262027b14fe 100644 --- a/ydb/core/tx/datashard/datashard_common_upload.h +++ b/ydb/core/tx/datashard/datashard_common_upload.h @@ -1,34 +1,34 @@ -#pragma once - -#include "datashard_impl.h" - +#pragma once + +#include "datashard_impl.h" + #include <ydb/core/engine/minikql/change_collector_iface.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -template <typename TEvRequest, typename TEvResponse> -class TCommonUploadOps { - using IChangeCollector = NMiniKQL::IChangeCollector; - - const bool BreakLocks; - const bool CollectChanges; - - typename TEvRequest::TPtr Ev; - THolder<TEvResponse> Result; - THolder<IChangeCollector> ChangeCollector; - -public: - explicit TCommonUploadOps(typename TEvRequest::TPtr& ev, bool breakLocks, bool collectChanges); - -protected: - bool Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion); + +template <typename TEvRequest, typename TEvResponse> +class TCommonUploadOps { + using IChangeCollector = NMiniKQL::IChangeCollector; + + const bool BreakLocks; + const bool CollectChanges; + + typename TEvRequest::TPtr Ev; + THolder<TEvResponse> Result; + THolder<IChangeCollector> ChangeCollector; + +public: + explicit TCommonUploadOps(typename TEvRequest::TPtr& ev, bool breakLocks, bool collectChanges); + +protected: + bool Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion); void SendResult(TDataShard* self, const TActorContext& ctx); - TVector<IChangeCollector::TChange> GetCollectedChanges() const; - -private: - void SetError(ui32 status, const TString& descr); -}; - + TVector<IChangeCollector::TChange> GetCollectedChanges() const; + +private: + void SetError(ui32 status, const TString& descr); +}; + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_erase.cpp b/ydb/core/tx/datashard/datashard_direct_erase.cpp index ffb1daed8cf..826819596fa 100644 --- a/ydb/core/tx/datashard/datashard_direct_erase.cpp +++ b/ydb/core/tx/datashard/datashard_direct_erase.cpp @@ -1,229 +1,229 @@ -#include "change_collector.h" -#include "datashard_direct_erase.h" -#include "erase_rows_condition.h" - +#include "change_collector.h" +#include "datashard_direct_erase.h" +#include "erase_rows_condition.h" + #include <ydb/core/base/appdata.h> - -#include <util/generic/xrange.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -TDirectTxErase::TDirectTxErase(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) - : Ev(ev) -{ -} - -TDirectTxErase::EStatus TDirectTxErase::CheckedExecute( - TDataShard* self, const TExecuteParams& params, - const NKikimrTxDataShard::TEvEraseRowsRequest& request, - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error) -{ - const ui64 tableId = request.GetTableId(); - const TTableId fullTableId(self->GetPathOwnerId(), tableId); - const ui64 localTableId = self->GetLocalTableId(fullTableId); - if (localTableId == 0) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; - error = TStringBuilder() << "Unknown table id: " << tableId; - return EStatus::Error; - } - + +TDirectTxErase::TDirectTxErase(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) + : Ev(ev) +{ +} + +TDirectTxErase::EStatus TDirectTxErase::CheckedExecute( + TDataShard* self, const TExecuteParams& params, + const NKikimrTxDataShard::TEvEraseRowsRequest& request, + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error) +{ + const ui64 tableId = request.GetTableId(); + const TTableId fullTableId(self->GetPathOwnerId(), tableId); + const ui64 localTableId = self->GetLocalTableId(fullTableId); + if (localTableId == 0) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; + error = TStringBuilder() << "Unknown table id: " << tableId; + return EStatus::Error; + } + const TUserTable& tableInfo = *self->GetUserTables().at(tableId); - Y_VERIFY(tableInfo.LocalTid == localTableId); - - if (request.GetSchemaVersion() && tableInfo.GetTableSchemaVersion() - && request.GetSchemaVersion() != tableInfo.GetTableSchemaVersion()) { - - status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; - error = TStringBuilder() << "Schema version mismatch" - << ": got " << request.GetSchemaVersion() - << ", expected " << tableInfo.GetTableSchemaVersion(); - return EStatus::Error; - } - - if (request.KeyColumnIdsSize() != tableInfo.KeyColumnIds.size()) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; - error = TStringBuilder() << "Key column count mismatch" - << ": got " << request.KeyColumnIdsSize() - << ", expected " << tableInfo.KeyColumnIds.size(); - return EStatus::Error; - } - - for (size_t i = 0; i < tableInfo.KeyColumnIds.size(); ++i) { - if (request.GetKeyColumnIds(i) != tableInfo.KeyColumnIds[i]) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; - error = TStringBuilder() << "Key column schema mismatch at position: " << i; - return EStatus::Error; - } - } - - THolder<IEraseRowsCondition> condition; - if (params) { - condition.Reset(CreateEraseRowsCondition(request)); - if (condition) { - condition->Prepare(params.Txc->DB.GetRowScheme(localTableId), 0); - } - - params.Tx->ChangeCollector.Reset(CreateChangeCollector(*self, params.Txc->DB, tableInfo, true)); - params.Tx->ChangeCollector->SetWriteVersion(params.WriteVersion); - if (params.Tx->ChangeCollector->NeedToReadKeys()) { - params.Tx->ChangeCollector->SetReadVersion(params.ReadVersion); - } - } - - bool pageFault = false; - for (const auto& serializedKey : request.GetKeyColumns()) { - TSerializedCellVec keyCells; - if (!TSerializedCellVec::TryParse(serializedKey, keyCells)) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; - error = "Cannot parse key"; - return EStatus::Error; - } - - if (keyCells.GetCells().size() != tableInfo.KeyColumnTypes.size()) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; - error = "Cell count doesn't match row scheme"; - return EStatus::Error; - } - - ui64 keyBytes = 0; - TVector<TRawTypeValue> key; - for (size_t ki : xrange(tableInfo.KeyColumnTypes.size())) { - const auto& kt = tableInfo.KeyColumnTypes[ki]; - const TCell& cell = keyCells.GetCells()[ki]; - - if (kt == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; - error = "Keys with Uint8 column values >127 are currently prohibited"; - return EStatus::Error; - } - - keyBytes += cell.Size(); - key.emplace_back(TRawTypeValue(cell.AsRef(), kt)); - } - - if (keyBytes > NLimits::MaxWriteKeySize) { - status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; - error = TStringBuilder() << "Key is too big" - << ": actual " << keyBytes << " bytes" - << ", limit " << NLimits::MaxWriteKeySize << " bytes"; - return EStatus::Error; - } - - if (!params) { - continue; - } - - if (condition) { - NTable::TRowState row; - const auto ready = params.Txc->DB.Select(localTableId, key, condition->Tags(), row, 0, params.ReadVersion); - - switch (ready) { - case NTable::EReady::Page: - pageFault = true; - break; - case NTable::EReady::Gone: - continue; - case NTable::EReady::Data: - if (!condition->Check(row)) { - continue; - } - break; - } - } - - Y_VERIFY(params.Tx->ChangeCollector); + Y_VERIFY(tableInfo.LocalTid == localTableId); + + if (request.GetSchemaVersion() && tableInfo.GetTableSchemaVersion() + && request.GetSchemaVersion() != tableInfo.GetTableSchemaVersion()) { + + status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; + error = TStringBuilder() << "Schema version mismatch" + << ": got " << request.GetSchemaVersion() + << ", expected " << tableInfo.GetTableSchemaVersion(); + return EStatus::Error; + } + + if (request.KeyColumnIdsSize() != tableInfo.KeyColumnIds.size()) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; + error = TStringBuilder() << "Key column count mismatch" + << ": got " << request.KeyColumnIdsSize() + << ", expected " << tableInfo.KeyColumnIds.size(); + return EStatus::Error; + } + + for (size_t i = 0; i < tableInfo.KeyColumnIds.size(); ++i) { + if (request.GetKeyColumnIds(i) != tableInfo.KeyColumnIds[i]) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; + error = TStringBuilder() << "Key column schema mismatch at position: " << i; + return EStatus::Error; + } + } + + THolder<IEraseRowsCondition> condition; + if (params) { + condition.Reset(CreateEraseRowsCondition(request)); + if (condition) { + condition->Prepare(params.Txc->DB.GetRowScheme(localTableId), 0); + } + + params.Tx->ChangeCollector.Reset(CreateChangeCollector(*self, params.Txc->DB, tableInfo, true)); + params.Tx->ChangeCollector->SetWriteVersion(params.WriteVersion); + if (params.Tx->ChangeCollector->NeedToReadKeys()) { + params.Tx->ChangeCollector->SetReadVersion(params.ReadVersion); + } + } + + bool pageFault = false; + for (const auto& serializedKey : request.GetKeyColumns()) { + TSerializedCellVec keyCells; + if (!TSerializedCellVec::TryParse(serializedKey, keyCells)) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; + error = "Cannot parse key"; + return EStatus::Error; + } + + if (keyCells.GetCells().size() != tableInfo.KeyColumnTypes.size()) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR; + error = "Cell count doesn't match row scheme"; + return EStatus::Error; + } + + ui64 keyBytes = 0; + TVector<TRawTypeValue> key; + for (size_t ki : xrange(tableInfo.KeyColumnTypes.size())) { + const auto& kt = tableInfo.KeyColumnTypes[ki]; + const TCell& cell = keyCells.GetCells()[ki]; + + if (kt == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; + error = "Keys with Uint8 column values >127 are currently prohibited"; + return EStatus::Error; + } + + keyBytes += cell.Size(); + key.emplace_back(TRawTypeValue(cell.AsRef(), kt)); + } + + if (keyBytes > NLimits::MaxWriteKeySize) { + status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; + error = TStringBuilder() << "Key is too big" + << ": actual " << keyBytes << " bytes" + << ", limit " << NLimits::MaxWriteKeySize << " bytes"; + return EStatus::Error; + } + + if (!params) { + continue; + } + + if (condition) { + NTable::TRowState row; + const auto ready = params.Txc->DB.Select(localTableId, key, condition->Tags(), row, 0, params.ReadVersion); + + switch (ready) { + case NTable::EReady::Page: + pageFault = true; + break; + case NTable::EReady::Gone: + continue; + case NTable::EReady::Data: + if (!condition->Check(row)) { + continue; + } + break; + } + } + + Y_VERIFY(params.Tx->ChangeCollector); if (!params.Tx->ChangeCollector->Collect(fullTableId, NTable::ERowOp::Erase, key, {})) { - pageFault = true; - } - - if (pageFault) { - continue; - } - - self->SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); + pageFault = true; + } + + if (pageFault) { + continue; + } + + self->SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); params.Txc->DB.Update(localTableId, NTable::ERowOp::Erase, key, {}, params.WriteVersion); - } - - if (pageFault) { - if (params && params.Tx->ChangeCollector) { - params.Tx->ChangeCollector->Reset(); - } - - return EStatus::PageFault; - } - - status = NKikimrTxDataShard::TEvEraseRowsResponse::OK; - return EStatus::Success; -} - + } + + if (pageFault) { + if (params && params.Tx->ChangeCollector) { + params.Tx->ChangeCollector->Reset(); + } + + return EStatus::PageFault; + } + + status = NKikimrTxDataShard::TEvEraseRowsResponse::OK; + return EStatus::Success; +} + bool TDirectTxErase::CheckRequest(TDataShard* self, const NKikimrTxDataShard::TEvEraseRowsRequest& request, - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error) -{ - const auto result = CheckedExecute(self, TExecuteParams::ForCheck(), request, status, error); - switch (result) { - case EStatus::Success: - return true; - case EStatus::Error: - return false; - case EStatus::PageFault: - Y_FAIL("Unexpected"); - } -} - + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error) +{ + const auto result = CheckedExecute(self, TExecuteParams::ForCheck(), request, status, error); + switch (result) { + case EStatus::Success: + return true; + case EStatus::Error: + return false; + case EStatus::PageFault: + Y_FAIL("Unexpected"); + } +} + bool TDirectTxErase::Execute(TDataShard* self, TTransactionContext& txc, - const TRowVersion& readVersion, const TRowVersion& writeVersion) -{ - const auto& record = Ev->Get()->Record; - + const TRowVersion& readVersion, const TRowVersion& writeVersion) +{ + const auto& record = Ev->Get()->Record; + Result = MakeHolder<TEvDataShard::TEvEraseRowsResponse>(); - Result->Record.SetTabletID(self->TabletID()); - - const auto params = TExecuteParams::ForExecute(this, &txc, readVersion, writeVersion); - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus status; - TString error; - - const auto result = CheckedExecute(self, params, record, status, error); - switch (result) { - case EStatus::Success: - case EStatus::Error: - break; - case EStatus::PageFault: - return false; - } - - Result->Record.SetStatus(status); - Result->Record.SetErrorDescription(error); - - self->IncCounter(COUNTER_ERASE_ROWS, record.GetKeyColumns().size()); - if (self->GetUserTables().contains(record.GetTableId())) { - self->GetUserTables().at(record.GetTableId())->Stats.UpdateTime = TAppData::TimeProvider->Now(); - } - - return true; -} - + Result->Record.SetTabletID(self->TabletID()); + + const auto params = TExecuteParams::ForExecute(this, &txc, readVersion, writeVersion); + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus status; + TString error; + + const auto result = CheckedExecute(self, params, record, status, error); + switch (result) { + case EStatus::Success: + case EStatus::Error: + break; + case EStatus::PageFault: + return false; + } + + Result->Record.SetStatus(status); + Result->Record.SetErrorDescription(error); + + self->IncCounter(COUNTER_ERASE_ROWS, record.GetKeyColumns().size()); + if (self->GetUserTables().contains(record.GetTableId())) { + self->GetUserTables().at(record.GetTableId())->Stats.UpdateTime = TAppData::TimeProvider->Now(); + } + + return true; +} + void TDirectTxErase::SendResult(TDataShard* self, const TActorContext& ctx) { - Y_VERIFY(Result); - - if (Result->Record.GetStatus() == NKikimrTxDataShard::TEvEraseRowsResponse::OK) { - self->IncCounter(COUNTER_ERASE_ROWS_SUCCESS); - } else { - self->IncCounter(COUNTER_ERASE_ROWS_ERROR); - } - - ctx.Send(Ev->Sender, std::move(Result)); -} - -TVector<NMiniKQL::IChangeCollector::TChange> TDirectTxErase::GetCollectedChanges() const { - if (!ChangeCollector) { - return {}; - } - - auto changes = std::move(ChangeCollector->GetCollected()); - return changes; -} - + Y_VERIFY(Result); + + if (Result->Record.GetStatus() == NKikimrTxDataShard::TEvEraseRowsResponse::OK) { + self->IncCounter(COUNTER_ERASE_ROWS_SUCCESS); + } else { + self->IncCounter(COUNTER_ERASE_ROWS_ERROR); + } + + ctx.Send(Ev->Sender, std::move(Result)); +} + +TVector<NMiniKQL::IChangeCollector::TChange> TDirectTxErase::GetCollectedChanges() const { + if (!ChangeCollector) { + return {}; + } + + auto changes = std::move(ChangeCollector->GetCollected()); + return changes; +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_erase.h b/ydb/core/tx/datashard/datashard_direct_erase.h index 803854a434a..689b868a36f 100644 --- a/ydb/core/tx/datashard/datashard_direct_erase.h +++ b/ydb/core/tx/datashard/datashard_direct_erase.h @@ -1,74 +1,74 @@ -#pragma once - -#include "datashard_direct_transaction.h" - -namespace NKikimr { +#pragma once + +#include "datashard_direct_transaction.h" + +namespace NKikimr { namespace NDataShard { - -class TDirectTxErase : public IDirectTx { - using IChangeCollector = NMiniKQL::IChangeCollector; - - TEvDataShard::TEvEraseRowsRequest::TPtr Ev; - THolder<TEvDataShard::TEvEraseRowsResponse> Result; - THolder<IChangeCollector> ChangeCollector; - - enum class EStatus { - Success, - Error, - PageFault, - }; - - struct TExecuteParams { - TDirectTxErase* const Tx; - TTransactionContext* const Txc; - const TRowVersion ReadVersion; - const TRowVersion WriteVersion; - - private: - explicit TExecuteParams(TDirectTxErase* tx, TTransactionContext* txc, - const TRowVersion& readVersion, const TRowVersion& writeVersion) - : Tx(tx) - , Txc(txc) - , ReadVersion(readVersion) - , WriteVersion(writeVersion) - { - } - - public: - static TExecuteParams ForCheck() { - return TExecuteParams(nullptr, nullptr, TRowVersion(), TRowVersion()); - } - - template <typename... Args> - static TExecuteParams ForExecute(Args&&... args) { - return TExecuteParams(std::forward<Args>(args)...); - } - - explicit operator bool() const { - if (!Tx || !Txc) { - Y_VERIFY(!Tx && !Txc); - return false; - } - - return true; - } - }; - - static EStatus CheckedExecute( - TDataShard* self, const TExecuteParams& params, - const NKikimrTxDataShard::TEvEraseRowsRequest& request, - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error); - -public: - explicit TDirectTxErase(TEvDataShard::TEvEraseRowsRequest::TPtr& ev); - + +class TDirectTxErase : public IDirectTx { + using IChangeCollector = NMiniKQL::IChangeCollector; + + TEvDataShard::TEvEraseRowsRequest::TPtr Ev; + THolder<TEvDataShard::TEvEraseRowsResponse> Result; + THolder<IChangeCollector> ChangeCollector; + + enum class EStatus { + Success, + Error, + PageFault, + }; + + struct TExecuteParams { + TDirectTxErase* const Tx; + TTransactionContext* const Txc; + const TRowVersion ReadVersion; + const TRowVersion WriteVersion; + + private: + explicit TExecuteParams(TDirectTxErase* tx, TTransactionContext* txc, + const TRowVersion& readVersion, const TRowVersion& writeVersion) + : Tx(tx) + , Txc(txc) + , ReadVersion(readVersion) + , WriteVersion(writeVersion) + { + } + + public: + static TExecuteParams ForCheck() { + return TExecuteParams(nullptr, nullptr, TRowVersion(), TRowVersion()); + } + + template <typename... Args> + static TExecuteParams ForExecute(Args&&... args) { + return TExecuteParams(std::forward<Args>(args)...); + } + + explicit operator bool() const { + if (!Tx || !Txc) { + Y_VERIFY(!Tx && !Txc); + return false; + } + + return true; + } + }; + + static EStatus CheckedExecute( + TDataShard* self, const TExecuteParams& params, + const NKikimrTxDataShard::TEvEraseRowsRequest& request, + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error); + +public: + explicit TDirectTxErase(TEvDataShard::TEvEraseRowsRequest::TPtr& ev); + static bool CheckRequest(TDataShard* self, const NKikimrTxDataShard::TEvEraseRowsRequest& request, - NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error); - + NKikimrTxDataShard::TEvEraseRowsResponse::EStatus& status, TString& error); + bool Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion) override; void SendResult(TDataShard* self, const TActorContext& ctx) override; - TVector<IChangeCollector::TChange> GetCollectedChanges() const override; -}; - + TVector<IChangeCollector::TChange> GetCollectedChanges() const override; +}; + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_transaction.cpp b/ydb/core/tx/datashard/datashard_direct_transaction.cpp index 85de4688816..145f0936cd4 100644 --- a/ydb/core/tx/datashard/datashard_direct_transaction.cpp +++ b/ydb/core/tx/datashard/datashard_direct_transaction.cpp @@ -1,22 +1,22 @@ #include "datashard_direct_transaction.h" -#include "datashard_direct_erase.h" -#include "datashard_direct_upload.h" +#include "datashard_direct_erase.h" +#include "datashard_direct_upload.h" namespace NKikimr { namespace NDataShard { -TDirectTransaction::TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvUploadRowsRequest::TPtr& ev) - : TOperation(TBasicOpInfo(txId, EOperationKind::DirectTx, Flags, 0, receivedAt, tieBreakerIndex)) - , Impl(new TDirectTxUpload(ev)) -{ -} - -TDirectTransaction::TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvEraseRowsRequest::TPtr& ev) - : TOperation(TBasicOpInfo(txId, EOperationKind::DirectTx, Flags, 0, receivedAt, tieBreakerIndex)) - , Impl(new TDirectTxErase(ev)) -{ -} - +TDirectTransaction::TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvUploadRowsRequest::TPtr& ev) + : TOperation(TBasicOpInfo(txId, EOperationKind::DirectTx, Flags, 0, receivedAt, tieBreakerIndex)) + , Impl(new TDirectTxUpload(ev)) +{ +} + +TDirectTransaction::TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvEraseRowsRequest::TPtr& ev) + : TOperation(TBasicOpInfo(txId, EOperationKind::DirectTx, Flags, 0, receivedAt, tieBreakerIndex)) + , Impl(new TDirectTxErase(ev)) +{ +} + void TDirectTransaction::BuildExecutionPlan(bool loaded) { Y_VERIFY(GetExecutionPlan().empty()); @@ -24,7 +24,7 @@ void TDirectTransaction::BuildExecutionPlan(bool loaded) TVector<EExecutionUnitKind> plan; plan.push_back(EExecutionUnitKind::BuildAndWaitDependencies); - plan.push_back(EExecutionUnitKind::DirectOp); + plan.push_back(EExecutionUnitKind::DirectOp); plan.push_back(EExecutionUnitKind::CompletedOperations); RewriteExecutionPlan(plan); @@ -40,12 +40,12 @@ bool TDirectTransaction::Execute(TDataShard* self, TTransactionContext& txc) { } void TDirectTransaction::SendResult(TDataShard* self, const TActorContext& ctx) { - Impl->SendResult(self, ctx); + Impl->SendResult(self, ctx); +} + +TVector<NMiniKQL::IChangeCollector::TChange> TDirectTransaction::GetCollectedChanges() const { + return Impl->GetCollectedChanges(); } -TVector<NMiniKQL::IChangeCollector::TChange> TDirectTransaction::GetCollectedChanges() const { - return Impl->GetCollectedChanges(); -} - } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_transaction.h b/ydb/core/tx/datashard/datashard_direct_transaction.h index e6b92a58ba3..5b42e1f1d71 100644 --- a/ydb/core/tx/datashard/datashard_direct_transaction.h +++ b/ydb/core/tx/datashard/datashard_direct_transaction.h @@ -11,32 +11,32 @@ namespace NKikimr { namespace NDataShard { -class IDirectTx { +class IDirectTx { public: - virtual ~IDirectTx() = default; + virtual ~IDirectTx() = default; virtual bool Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion) = 0; virtual void SendResult(TDataShard* self, const TActorContext& ctx) = 0; - virtual TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const = 0; -}; + virtual TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const = 0; +}; -class TDirectTransaction : public TOperation { +class TDirectTransaction : public TOperation { public: - TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvUploadRowsRequest::TPtr& ev); - TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvEraseRowsRequest::TPtr& ev); + TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvUploadRowsRequest::TPtr& ev); + TDirectTransaction(ui64 txId, TInstant receivedAt, ui64 tieBreakerIndex, TEvDataShard::TEvEraseRowsRequest::TPtr& ev); void BuildExecutionPlan(bool) override; private: bool Execute(TDataShard* self, TTransactionContext& txc); void SendResult(TDataShard* self, const TActorContext& ctx); - TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const; + TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const; - friend class TDirectOpUnit; + friend class TDirectOpUnit; -private: - THolder<IDirectTx> Impl; - static constexpr ui32 Flags = NTxDataShard::TTxFlags::Immediate | NTxDataShard::TTxFlags::GlobalWriter; +private: + THolder<IDirectTx> Impl; + static constexpr ui32 Flags = NTxDataShard::TTxFlags::Immediate | NTxDataShard::TTxFlags::GlobalWriter; }; } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_upload.cpp b/ydb/core/tx/datashard/datashard_direct_upload.cpp index e0cc1c28e0d..6173c0e218f 100644 --- a/ydb/core/tx/datashard/datashard_direct_upload.cpp +++ b/ydb/core/tx/datashard/datashard_direct_upload.cpp @@ -1,24 +1,24 @@ -#include "datashard_direct_upload.h" +#include "datashard_direct_upload.h" namespace NKikimr { namespace NDataShard { -TDirectTxUpload::TDirectTxUpload(TEvDataShard::TEvUploadRowsRequest::TPtr& ev) - : TCommonUploadOps(ev, true, true) -{ -} - +TDirectTxUpload::TDirectTxUpload(TEvDataShard::TEvUploadRowsRequest::TPtr& ev) + : TCommonUploadOps(ev, true, true) +{ +} + bool TDirectTxUpload::Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion) { - return TCommonUploadOps::Execute(self, txc, readVersion, writeVersion); + return TCommonUploadOps::Execute(self, txc, readVersion, writeVersion); } void TDirectTxUpload::SendResult(TDataShard* self, const TActorContext& ctx) { - TCommonUploadOps::SendResult(self, ctx); + TCommonUploadOps::SendResult(self, ctx); +} + +TVector<NMiniKQL::IChangeCollector::TChange> TDirectTxUpload::GetCollectedChanges() const { + return TCommonUploadOps::GetCollectedChanges(); } -TVector<NMiniKQL::IChangeCollector::TChange> TDirectTxUpload::GetCollectedChanges() const { - return TCommonUploadOps::GetCollectedChanges(); -} - } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_direct_upload.h b/ydb/core/tx/datashard/datashard_direct_upload.h index d9ae3f61724..7ca84f19a7b 100644 --- a/ydb/core/tx/datashard/datashard_direct_upload.h +++ b/ydb/core/tx/datashard/datashard_direct_upload.h @@ -1,22 +1,22 @@ -#pragma once - -#include "datashard_common_upload.h" -#include "datashard_direct_transaction.h" - -namespace NKikimr { +#pragma once + +#include "datashard_common_upload.h" +#include "datashard_direct_transaction.h" + +namespace NKikimr { namespace NDataShard { - -class TDirectTxUpload : public IDirectTx, - public TCommonUploadOps< - TEvDataShard::TEvUploadRowsRequest, - TEvDataShard::TEvUploadRowsResponse> { -public: - explicit TDirectTxUpload(TEvDataShard::TEvUploadRowsRequest::TPtr& ev); - + +class TDirectTxUpload : public IDirectTx, + public TCommonUploadOps< + TEvDataShard::TEvUploadRowsRequest, + TEvDataShard::TEvUploadRowsResponse> { +public: + explicit TDirectTxUpload(TEvDataShard::TEvUploadRowsRequest::TPtr& ev); + bool Execute(TDataShard* self, TTransactionContext& txc, const TRowVersion& readVersion, const TRowVersion& writeVersion) override; void SendResult(TDataShard* self, const TActorContext& ctx) override; - TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const override; -}; - + TVector<NMiniKQL::IChangeCollector::TChange> GetCollectedChanges() const override; +}; + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_distributed_erase.cpp b/ydb/core/tx/datashard/datashard_distributed_erase.cpp index 3a7be8cd7a4..fae7d2edeb3 100644 --- a/ydb/core/tx/datashard/datashard_distributed_erase.cpp +++ b/ydb/core/tx/datashard/datashard_distributed_erase.cpp @@ -1,6 +1,6 @@ -#include "datashard_distributed_erase.h" -#include "datashard_impl.h" - +#include "datashard_distributed_erase.h" +#include "datashard_impl.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/protos/services.pb.h> @@ -8,106 +8,106 @@ #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/tx.h> #include <ydb/library/aclib/aclib.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + #include <ydb/library/yql/public/udf/udf_types.h> - -#include <util/generic/bitmap.h> -#include <util/generic/hash_set.h> -#include <util/generic/ptr.h> -#include <util/string/builder.h> - -#if defined LOG_T || \ - defined LOG_D || \ - defined LOG_I || \ - defined LOG_N || \ - defined LOG_W || \ - defined LOG_E || \ - defined LOG_C -#error log macro redefinition -#endif - -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) - -namespace NKikimr { + +#include <util/generic/bitmap.h> +#include <util/generic/hash_set.h> +#include <util/generic/ptr.h> +#include <util/string/builder.h> + +#if defined LOG_T || \ + defined LOG_D || \ + defined LOG_I || \ + defined LOG_N || \ + defined LOG_W || \ + defined LOG_E || \ + defined LOG_C +#error log macro redefinition +#endif + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) +#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "[DistEraser] " << SelfId() << " " << stream) + +namespace NKikimr { namespace NDataShard { - -class TDistEraser: public TActorBootstrapped<TDistEraser> { - using TEvResponse = NKikimrTxDataShard::TEvEraseRowsResponse; - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TResolve = NSchemeCache::TSchemeCacheRequest; - using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; - - class TTableInfo { - template <typename T, typename TExtractor> - static TVector<T> MakeKeyColumnSmth(const TNavigate::TEntry& entry, TExtractor extractor) { - TVector<T> result; - - for (const auto& [_, column] : entry.Columns) { - if (column.KeyOrder < 0) { - continue; - } - - if (result.size() <= static_cast<ui32>(column.KeyOrder)) { - result.resize(column.KeyOrder + 1); - } - - result[column.KeyOrder] = extractor(column); - } - - return result; - } - - static TKeyMap MakeSelfKeyMap(const TNavigate::TEntry& entry) { - return MakeKeyColumnSmth<std::pair<ui32, ui32>>(entry, [](auto column) { - return std::make_pair(column.Id, column.Id); - }); - } - - static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { - return MakeKeyColumnSmth<NUdf::TDataTypeId>(entry, [](auto column) { - return column.PType; - }); - } - - static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { - TVector<TCell> fromValues(keyColumnsCount); - TVector<TCell> toValues; - return TSerializedTableRange(fromValues, true, toValues, false); - } - - static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { - const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); - return MakeHolder<TKeyDesc>( - entry.TableId, - GetFullRange(keyColumnTypes.size()).ToTableRange(), - TKeyDesc::ERowOperation::Erase, - keyColumnTypes, - TVector<TKeyDesc::TColumnOp>() - ); - } - - public: - explicit TTableInfo(const TNavigate::TEntry& entry, const TKeyMap& keyMap) - : KeyMap(keyMap ? keyMap : MakeSelfKeyMap(entry)) - , KeyDesc(MakeKeyDesc(entry)) - { - } - - const TKeyMap& GetKeyMap() const { - return KeyMap; - } - + +class TDistEraser: public TActorBootstrapped<TDistEraser> { + using TEvResponse = NKikimrTxDataShard::TEvEraseRowsResponse; + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TResolve = NSchemeCache::TSchemeCacheRequest; + using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; + + class TTableInfo { + template <typename T, typename TExtractor> + static TVector<T> MakeKeyColumnSmth(const TNavigate::TEntry& entry, TExtractor extractor) { + TVector<T> result; + + for (const auto& [_, column] : entry.Columns) { + if (column.KeyOrder < 0) { + continue; + } + + if (result.size() <= static_cast<ui32>(column.KeyOrder)) { + result.resize(column.KeyOrder + 1); + } + + result[column.KeyOrder] = extractor(column); + } + + return result; + } + + static TKeyMap MakeSelfKeyMap(const TNavigate::TEntry& entry) { + return MakeKeyColumnSmth<std::pair<ui32, ui32>>(entry, [](auto column) { + return std::make_pair(column.Id, column.Id); + }); + } + + static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { + return MakeKeyColumnSmth<NUdf::TDataTypeId>(entry, [](auto column) { + return column.PType; + }); + } + + static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { + TVector<TCell> fromValues(keyColumnsCount); + TVector<TCell> toValues; + return TSerializedTableRange(fromValues, true, toValues, false); + } + + static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { + const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); + return MakeHolder<TKeyDesc>( + entry.TableId, + GetFullRange(keyColumnTypes.size()).ToTableRange(), + TKeyDesc::ERowOperation::Erase, + keyColumnTypes, + TVector<TKeyDesc::TColumnOp>() + ); + } + + public: + explicit TTableInfo(const TNavigate::TEntry& entry, const TKeyMap& keyMap) + : KeyMap(keyMap ? keyMap : MakeSelfKeyMap(entry)) + , KeyDesc(MakeKeyDesc(entry)) + { + } + + const TKeyMap& GetKeyMap() const { + return KeyMap; + } + THolder<TKeyDesc> TakeKeyDesc() { return std::move(KeyDesc); } @@ -116,1002 +116,1002 @@ class TDistEraser: public TActorBootstrapped<TDistEraser> { KeyDesc = std::move(keyDesc); } - const TKeyDesc* GetKeyDesc() const { - return KeyDesc.Get(); - } - + const TKeyDesc* GetKeyDesc() const { + return KeyDesc.Get(); + } + public: uintptr_t ToUserDataHandle() { return reinterpret_cast<uintptr_t>(this); - } - + } + static TTableInfo& FromUserDataHandle(uintptr_t userData) { return *reinterpret_cast<TTableInfo*>(userData); } - private: - const TKeyMap KeyMap; - THolder<TKeyDesc> KeyDesc; - - }; // TTableInfo - - class TShardKeys { - public: - void Add(TString&& key, size_t row) { - Keys.emplace_back(std::move(key)); - PresentRows.Set(row); - } - - TVector<TString>& GetKeys() { - return Keys; - } - - const TDynBitMap& GetPresentRows() const { - return PresentRows; - } - - private: - TVector<TString> Keys; - TDynBitMap PresentRows; - }; // TShardKeys - - void Reply(TEvResponse::EStatus status = TEvResponse::OK, const TString& error = TString()) { - const TString done = TStringBuilder() << "Reply" - << ": txId# " << TxId - << ", status# " << status - << ", error# " << error; - - if (status == TEvResponse::OK) { - LOG_D(done); - } else { - LOG_E(done); - } - - auto response = MakeHolder<TEvDataShard::TEvEraseRowsResponse>(); - auto& record = response->Record; - record.SetStatus(status); - record.SetErrorDescription(error); - - Send(ReplyTo, std::move(response)); - PassAway(); - } - - void SchemeError(const TString& error) { - Reply(TEvResponse::SCHEME_ERROR, error); - } - - void BadRequest(const TString& error) { - Reply(TEvResponse::BAD_REQUEST, error); - } - - void ExecError(const TString& error) { - Reply(TEvResponse::EXEC_ERROR, error); - } - - void CoordinatorDeclined(const TString& error) { - Reply(TEvResponse::COORDINATOR_DECLINED, error); - } - - void CoordinatorUnknown(const TString& error) { - Reply(TEvResponse::COORDINATOR_UNKNOWN, error); - } - - void ShardUnknown(const TString& error) { - Reply(TEvResponse::SHARD_UNKNOWN, error); - } - - void DeliveryProblem() { - Reply(TEvResponse::UNKNOWN, "Unexpected delivery problem"); - } - - template <typename T> - bool CheckNotEmpty(const TAutoPtr<T>& result, const TStringBuf marker) { - if (result) { - return true; - } - - SchemeError(TStringBuilder() << "Empty result from scheme cache at '" << marker << "'"); - return false; - } - - template <typename T> - bool CheckEntriesCount(const TAutoPtr<T>& result, const TStringBuf marker, ui32 expected) { - if (result->ResultSet.size() == expected) { - return true; - } - - SchemeError(TStringBuilder() << "Entries count mismatch at '" << marker << "'" - << ": expected# " << expected - << ", actual# " << result->ResultSet.size() - << ", result# " << result->ToString(*AppData()->TypeRegistry)); - return false; - } - - static bool IsSucceeded(TNavigate::EStatus status) { - return status == TNavigate::EStatus::Ok; - } - - static bool IsSucceeded(TResolve::EStatus status) { - return status == TResolve::EStatus::OkData; - } - - static TTableId GetTableId(const TNavigate::TEntry& entry) { - return entry.TableId; - } - - static TTableId GetTableId(const TResolve::TEntry& entry) { - return entry.KeyDescription->TableId; - } - - template <typename T> - bool CheckEntrySucceeded(const T& entry, const TStringBuf marker) { - if (IsSucceeded(entry.Status)) { - return true; - } - - SchemeError(TStringBuilder() << "Failed to resolve table at '" << marker << "'" - << ": tableId# " << GetTableId(entry) - << ", status# " << entry.Status); - return false; - } - - static TNavigate::TEntry MakeNavigateEntry(const TTableId& tableId) { - TNavigate::TEntry entry; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - entry.TableId = tableId; - entry.Operation = TNavigate::OpTable; - entry.ShowPrivatePath = true; - return entry; - } - - void AddTableInfo(const TNavigate::TEntry& entry, const TKeyMap& keyMap = {}) { - Y_VERIFY_DEBUG(!TableInfos.contains(entry.TableId)); - TableInfos.emplace(entry.TableId, TTableInfo(entry, keyMap)); - } - - static ui64 GetShardId(const TTableRange& range, const TKeyDesc* keyDesc) { - Y_VERIFY(range.Point); - Y_VERIFY(!keyDesc->Partitions.empty()); - - TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( - keyDesc->Partitions.begin(), keyDesc->Partitions.end(), true, - [&](const TKeyDesc::TPartitionInfo& partition, bool) { - const int compares = CompareBorders<true, false>( - partition.Range->EndKeyPrefix.GetCells(), range.From, - partition.Range->IsInclusive || partition.Range->IsPoint, - range.InclusiveFrom || range.Point, keyDesc->KeyColumnTypes - ); - - return (compares < 0); - } - ); - - Y_VERIFY(it != keyDesc->Partitions.end()); - return it->ShardId; - } - - void CancelProposal(ui64 exceptShardId = 0) { - for (const ui64 shardId : Shards) { - if (shardId == exceptShardId) { - continue; - } - - LOG_D("Cancel proposal" - << ": txId# " << TxId - << ", shard# " << shardId); - - auto cancel = MakeHolder<TEvDataShard::TEvCancelTransactionProposal>(TxId); + private: + const TKeyMap KeyMap; + THolder<TKeyDesc> KeyDesc; + + }; // TTableInfo + + class TShardKeys { + public: + void Add(TString&& key, size_t row) { + Keys.emplace_back(std::move(key)); + PresentRows.Set(row); + } + + TVector<TString>& GetKeys() { + return Keys; + } + + const TDynBitMap& GetPresentRows() const { + return PresentRows; + } + + private: + TVector<TString> Keys; + TDynBitMap PresentRows; + }; // TShardKeys + + void Reply(TEvResponse::EStatus status = TEvResponse::OK, const TString& error = TString()) { + const TString done = TStringBuilder() << "Reply" + << ": txId# " << TxId + << ", status# " << status + << ", error# " << error; + + if (status == TEvResponse::OK) { + LOG_D(done); + } else { + LOG_E(done); + } + + auto response = MakeHolder<TEvDataShard::TEvEraseRowsResponse>(); + auto& record = response->Record; + record.SetStatus(status); + record.SetErrorDescription(error); + + Send(ReplyTo, std::move(response)); + PassAway(); + } + + void SchemeError(const TString& error) { + Reply(TEvResponse::SCHEME_ERROR, error); + } + + void BadRequest(const TString& error) { + Reply(TEvResponse::BAD_REQUEST, error); + } + + void ExecError(const TString& error) { + Reply(TEvResponse::EXEC_ERROR, error); + } + + void CoordinatorDeclined(const TString& error) { + Reply(TEvResponse::COORDINATOR_DECLINED, error); + } + + void CoordinatorUnknown(const TString& error) { + Reply(TEvResponse::COORDINATOR_UNKNOWN, error); + } + + void ShardUnknown(const TString& error) { + Reply(TEvResponse::SHARD_UNKNOWN, error); + } + + void DeliveryProblem() { + Reply(TEvResponse::UNKNOWN, "Unexpected delivery problem"); + } + + template <typename T> + bool CheckNotEmpty(const TAutoPtr<T>& result, const TStringBuf marker) { + if (result) { + return true; + } + + SchemeError(TStringBuilder() << "Empty result from scheme cache at '" << marker << "'"); + return false; + } + + template <typename T> + bool CheckEntriesCount(const TAutoPtr<T>& result, const TStringBuf marker, ui32 expected) { + if (result->ResultSet.size() == expected) { + return true; + } + + SchemeError(TStringBuilder() << "Entries count mismatch at '" << marker << "'" + << ": expected# " << expected + << ", actual# " << result->ResultSet.size() + << ", result# " << result->ToString(*AppData()->TypeRegistry)); + return false; + } + + static bool IsSucceeded(TNavigate::EStatus status) { + return status == TNavigate::EStatus::Ok; + } + + static bool IsSucceeded(TResolve::EStatus status) { + return status == TResolve::EStatus::OkData; + } + + static TTableId GetTableId(const TNavigate::TEntry& entry) { + return entry.TableId; + } + + static TTableId GetTableId(const TResolve::TEntry& entry) { + return entry.KeyDescription->TableId; + } + + template <typename T> + bool CheckEntrySucceeded(const T& entry, const TStringBuf marker) { + if (IsSucceeded(entry.Status)) { + return true; + } + + SchemeError(TStringBuilder() << "Failed to resolve table at '" << marker << "'" + << ": tableId# " << GetTableId(entry) + << ", status# " << entry.Status); + return false; + } + + static TNavigate::TEntry MakeNavigateEntry(const TTableId& tableId) { + TNavigate::TEntry entry; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.TableId = tableId; + entry.Operation = TNavigate::OpTable; + entry.ShowPrivatePath = true; + return entry; + } + + void AddTableInfo(const TNavigate::TEntry& entry, const TKeyMap& keyMap = {}) { + Y_VERIFY_DEBUG(!TableInfos.contains(entry.TableId)); + TableInfos.emplace(entry.TableId, TTableInfo(entry, keyMap)); + } + + static ui64 GetShardId(const TTableRange& range, const TKeyDesc* keyDesc) { + Y_VERIFY(range.Point); + Y_VERIFY(!keyDesc->Partitions.empty()); + + TVector<TKeyDesc::TPartitionInfo>::const_iterator it = LowerBound( + keyDesc->Partitions.begin(), keyDesc->Partitions.end(), true, + [&](const TKeyDesc::TPartitionInfo& partition, bool) { + const int compares = CompareBorders<true, false>( + partition.Range->EndKeyPrefix.GetCells(), range.From, + partition.Range->IsInclusive || partition.Range->IsPoint, + range.InclusiveFrom || range.Point, keyDesc->KeyColumnTypes + ); + + return (compares < 0); + } + ); + + Y_VERIFY(it != keyDesc->Partitions.end()); + return it->ShardId; + } + + void CancelProposal(ui64 exceptShardId = 0) { + for (const ui64 shardId : Shards) { + if (shardId == exceptShardId) { + continue; + } + + LOG_D("Cancel proposal" + << ": txId# " << TxId + << ", shard# " << shardId); + + auto cancel = MakeHolder<TEvDataShard::TEvCancelTransactionProposal>(TxId); Send(LeaderPipeCache, new TEvPipeCache::TEvForward(cancel.Release(), shardId, false)); - } - } - - /// Allocate tx id - - void AllocateTxId() { - LOG_D("AllocateTxId"); - - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); - Become(&TThis::StateAllocateTxId); - } - - STATEFN(StateAllocateTxId) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); - default: - return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { - LOG_D("Handle TEvTxUserProxy::TEvAllocateTxIdResult"); - - TxId = ev->Get()->TxId; + } + } + + /// Allocate tx id + + void AllocateTxId() { + LOG_D("AllocateTxId"); + + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); + Become(&TThis::StateAllocateTxId); + } + + STATEFN(StateAllocateTxId) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); + default: + return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { + LOG_D("Handle TEvTxUserProxy::TEvAllocateTxIdResult"); + + TxId = ev->Get()->TxId; LeaderPipeCache = ev->Get()->Services.LeaderPipeCache; - Mon = ev->Get()->TxProxyMon; - - ResolveTables(); - } - - /// Resolve tables - - void ResolveTables() { - LOG_D("Resolve tables" - << ": txId# " << TxId); - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(MakeNavigateEntry(MainTableId)); - for (const auto& [tableId, _] : Indexes) { - request->ResultSet.emplace_back(MakeNavigateEntry(tableId)); - } - - Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); - Become(&TThis::StateResolveTables); - } - - STATEFN(StateResolveTables) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - default: - return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const auto& request = ev->Get()->Request; + Mon = ev->Get()->TxProxyMon; + + ResolveTables(); + } + + /// Resolve tables + + void ResolveTables() { + LOG_D("Resolve tables" + << ": txId# " << TxId); + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(MakeNavigateEntry(MainTableId)); + for (const auto& [tableId, _] : Indexes) { + request->ResultSet.emplace_back(MakeNavigateEntry(tableId)); + } + + Send(MakeSchemeCacheID(), new TEvNavigate(request.Release())); + Become(&TThis::StateResolveTables); + } + + STATEFN(StateResolveTables) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + default: + return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; const TStringBuf marker = "ResolveTables"; - - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(request, marker)) { - return; - } - - if (!CheckEntriesCount(request, marker, 1 /* main */ + Indexes.size())) { - return; - } - - // main - { - const auto& entry = request->ResultSet.at(0); - if (!CheckEntrySucceeded(entry, marker)) { - return; - } - - if (entry.TableId.PathId != MainTableId.PathId) { - return SchemeError(TStringBuilder() << "Main table's path id mismatch" - << ": expected# " << MainTableId.PathId - << ", actual# " << entry.TableId.PathId - << ", entry# " << entry.ToString()); - } - - if (entry.TableId.SchemaVersion != MainTableId.SchemaVersion) { - return SchemeError(TStringBuilder() << "Main table's schema version mismatch" - << ": expected# " << MainTableId.SchemaVersion - << ", actual# " << entry.TableId.SchemaVersion - << ", entry# " << entry.ToString()); - } - - ui32 nIndexes = 0; - for (const auto& index : entry.Indexes) { - if (index.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync) { - continue; - } - - const auto tableId = TTableId(index.GetPathOwnerId(), index.GetLocalPathId(), index.GetSchemaVersion()); - switch (index.GetState()) { + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(request, marker)) { + return; + } + + if (!CheckEntriesCount(request, marker, 1 /* main */ + Indexes.size())) { + return; + } + + // main + { + const auto& entry = request->ResultSet.at(0); + if (!CheckEntrySucceeded(entry, marker)) { + return; + } + + if (entry.TableId.PathId != MainTableId.PathId) { + return SchemeError(TStringBuilder() << "Main table's path id mismatch" + << ": expected# " << MainTableId.PathId + << ", actual# " << entry.TableId.PathId + << ", entry# " << entry.ToString()); + } + + if (entry.TableId.SchemaVersion != MainTableId.SchemaVersion) { + return SchemeError(TStringBuilder() << "Main table's schema version mismatch" + << ": expected# " << MainTableId.SchemaVersion + << ", actual# " << entry.TableId.SchemaVersion + << ", entry# " << entry.ToString()); + } + + ui32 nIndexes = 0; + for (const auto& index : entry.Indexes) { + if (index.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync) { + continue; + } + + const auto tableId = TTableId(index.GetPathOwnerId(), index.GetLocalPathId(), index.GetSchemaVersion()); + switch (index.GetState()) { case NKikimrSchemeOp::EIndexStateReady: case NKikimrSchemeOp::EIndexStateWriteOnly: case NKikimrSchemeOp::EIndexStateNotReady: - break; + break; case NKikimrSchemeOp::EIndexStateInvalid: - return SchemeError(TStringBuilder() << "Invalid index state" - << ": tableId# " << tableId - << ", name# " << index.GetName()); - default: - return SchemeError(TStringBuilder() << "Unknown index state" - << ": tableId# " << tableId - << ", name# " << index.GetName() - << ", state# " << static_cast<ui32>(index.GetState())); - } - - ++nIndexes; - } - - if (nIndexes != Indexes.size()) { - return SchemeError(TStringBuilder() << "Indexes count mismatch" - << ": expected# " << Indexes.size() - << ", actual# " << entry.Indexes.size() - << ", entry# " << entry.ToString()); - } - - AddTableInfo(entry); - } - - // indexes - THashSet<TTableId> indexesSeen; - - for (ui32 i = 1; i < request->ResultSet.size(); ++i) { - const auto& entry = request->ResultSet.at(i); - if (!CheckEntrySucceeded(entry, marker)) { - return; - } - - if (!Indexes.contains(entry.TableId)) { - return SchemeError(TStringBuilder() << "Unknown index" - << " (probably schema version mismatch)" - << ": tableId# " << entry.TableId - << ", path# " << JoinPath(entry.Path)); - } - - if (!indexesSeen.insert(entry.TableId).second) { - return SchemeError(TStringBuilder() << "Duplicate index" - << ": tableId# " << entry.TableId - << ", path# " << JoinPath(entry.Path)); - } - - AddTableInfo(entry, Indexes.at(entry.TableId)); - } - - if (indexesSeen.size() != Indexes.size()) { - return SchemeError(TStringBuilder() << "Incomplete indexes" - << ": expected# " << Indexes.size() - << ", actual# " << indexesSeen.size()); - } - - NSchemeCache::TDomainInfo::TPtr domainInfo; - for (const auto& entry : request->ResultSet) { - if (!entry.DomainInfo) { - return SchemeError(TStringBuilder() << "Empty domain info" - << ": entry# " << entry.ToString()); - } - - if (!domainInfo) { - domainInfo = entry.DomainInfo; - continue; - } - - if (domainInfo->DomainKey != entry.DomainInfo->DomainKey) { - return SchemeError(TStringBuilder() << "Failed locality check" - << ": expected# " << domainInfo->DomainKey - << ", actual# " << entry.DomainInfo->DomainKey - << ", entry# " << entry.ToString()); - } - } - - Y_VERIFY(TxId); - SelectedCoordinator = domainInfo->Coordinators.Select(TxId); - - ResolveKeys(); - } - - /// Resolve keys - - void ResolveKeys() { - LOG_D("Resolve keys" - << ": txId# " << TxId); - - Y_VERIFY(!TableInfos.empty()); - - auto request = MakeHolder<TResolve>(); - for (auto& [_, info] : TableInfos) { + return SchemeError(TStringBuilder() << "Invalid index state" + << ": tableId# " << tableId + << ", name# " << index.GetName()); + default: + return SchemeError(TStringBuilder() << "Unknown index state" + << ": tableId# " << tableId + << ", name# " << index.GetName() + << ", state# " << static_cast<ui32>(index.GetState())); + } + + ++nIndexes; + } + + if (nIndexes != Indexes.size()) { + return SchemeError(TStringBuilder() << "Indexes count mismatch" + << ": expected# " << Indexes.size() + << ", actual# " << entry.Indexes.size() + << ", entry# " << entry.ToString()); + } + + AddTableInfo(entry); + } + + // indexes + THashSet<TTableId> indexesSeen; + + for (ui32 i = 1; i < request->ResultSet.size(); ++i) { + const auto& entry = request->ResultSet.at(i); + if (!CheckEntrySucceeded(entry, marker)) { + return; + } + + if (!Indexes.contains(entry.TableId)) { + return SchemeError(TStringBuilder() << "Unknown index" + << " (probably schema version mismatch)" + << ": tableId# " << entry.TableId + << ", path# " << JoinPath(entry.Path)); + } + + if (!indexesSeen.insert(entry.TableId).second) { + return SchemeError(TStringBuilder() << "Duplicate index" + << ": tableId# " << entry.TableId + << ", path# " << JoinPath(entry.Path)); + } + + AddTableInfo(entry, Indexes.at(entry.TableId)); + } + + if (indexesSeen.size() != Indexes.size()) { + return SchemeError(TStringBuilder() << "Incomplete indexes" + << ": expected# " << Indexes.size() + << ", actual# " << indexesSeen.size()); + } + + NSchemeCache::TDomainInfo::TPtr domainInfo; + for (const auto& entry : request->ResultSet) { + if (!entry.DomainInfo) { + return SchemeError(TStringBuilder() << "Empty domain info" + << ": entry# " << entry.ToString()); + } + + if (!domainInfo) { + domainInfo = entry.DomainInfo; + continue; + } + + if (domainInfo->DomainKey != entry.DomainInfo->DomainKey) { + return SchemeError(TStringBuilder() << "Failed locality check" + << ": expected# " << domainInfo->DomainKey + << ", actual# " << entry.DomainInfo->DomainKey + << ", entry# " << entry.ToString()); + } + } + + Y_VERIFY(TxId); + SelectedCoordinator = domainInfo->Coordinators.Select(TxId); + + ResolveKeys(); + } + + /// Resolve keys + + void ResolveKeys() { + LOG_D("Resolve keys" + << ": txId# " << TxId); + + Y_VERIFY(!TableInfos.empty()); + + auto request = MakeHolder<TResolve>(); + for (auto& [_, info] : TableInfos) { auto& entry = request->ResultSet.emplace_back(info.TakeKeyDesc()); entry.Access = NACLib::EAccessRights::EraseRow; entry.UserData = info.ToUserDataHandle(); - } - - Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); - Become(&TThis::StateResolveKeys); - - ResolvingKeys = true; - } - - STATEFN(StateResolveKeys) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, Handle); - default: - return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - ResolvingKeys = false; - - if (Cancelled) { - return PassAway(); - } - - const auto& request = ev->Get()->Request; + } + + Send(MakeSchemeCacheID(), new TEvResolve(request.Release())); + Become(&TThis::StateResolveKeys); + + ResolvingKeys = true; + } + + STATEFN(StateResolveKeys) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, Handle); + default: + return StatePrepareBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { + ResolvingKeys = false; + + if (Cancelled) { + return PassAway(); + } + + const auto& request = ev->Get()->Request; const TStringBuf marker = "ResolveKeys"; - - LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySetResult" - << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); - - if (!CheckNotEmpty(request, marker)) { - return; - } - - if (!CheckEntriesCount(request, marker, TableInfos.size())) { - return; - } - - for (ui32 i = 0; i < request->ResultSet.size(); ++i) { + + LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySetResult" + << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (!CheckNotEmpty(request, marker)) { + return; + } + + if (!CheckEntriesCount(request, marker, TableInfos.size())) { + return; + } + + for (ui32 i = 0; i < request->ResultSet.size(); ++i) { auto& entry = request->ResultSet.at(i); - if (!CheckEntrySucceeded(entry, marker)) { - return; - } - - if (entry.KeyDescription->Partitions.empty()) { - return SchemeError(TStringBuilder() << "Empty partitions list" - << ": entry# " << entry.ToString(*AppData()->TypeRegistry)); - } + if (!CheckEntrySucceeded(entry, marker)) { + return; + } + + if (entry.KeyDescription->Partitions.empty()) { + return SchemeError(TStringBuilder() << "Empty partitions list" + << ": entry# " << entry.ToString(*AppData()->TypeRegistry)); + } auto& info = TTableInfo::FromUserDataHandle(entry.UserData); info.SetKeyDesc(std::move(entry.KeyDescription)); - } - - if (Request) { - Handle(Request); - } else { - Become(&TThis::StateAwaitRequest); - } - } - - /// Process request - - STATEFN(StateAwaitRequest) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvDataShard::TEvEraseRowsRequest, Handle); - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - - void Store(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) { - LOG_D("Store TEvDataShard::TEvEraseRowsRequest"); - Request = ev; - } - - void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) { - LOG_D("Handle TEvDataShard::TEvEraseRowsRequest"); - - const auto& record = ev->Get()->Record; - - switch (record.GetConditionCase()) { - case NKikimrTxDataShard::TEvEraseRowsRequest::kExpiration: - break; - default: - return BadRequest(TStringBuilder() << "Unknown condition" - << ": condition# " << static_cast<ui32>(record.GetConditionCase())); - } - - THashMap<ui32, ui32> keyColumnIdToIdx; - for (ui32 i = 0; i < record.KeyColumnIdsSize(); ++i) { - Y_VERIFY_DEBUG(!keyColumnIdToIdx.contains(record.GetKeyColumnIds(i))); - keyColumnIdToIdx.emplace(record.GetKeyColumnIds(i), i); - } - - TVector<ui32> indexColumnIds; - { - Y_VERIFY(TableInfos.contains(MainTableId)); - const auto& mainTableInfo = TableInfos.at(MainTableId); - - THashSet<ui32> mainTableKeys; - for (const auto& [_, id] : mainTableInfo.GetKeyMap()) { - Y_VERIFY_DEBUG(!mainTableKeys.contains(id)); - mainTableKeys.insert(id); - } - - for (const auto& id : record.GetKeyColumnIds()) { - if (mainTableKeys.contains(id)) { - continue; - } - - indexColumnIds.push_back(id); - } - } - - THashMap<TTableId, THashMap<ui64, TShardKeys>> keys; // table to shard to keys - TVector<TString> indexColumnValues(Reserve(record.KeyColumnsSize())); - for (ui32 i = 0; i < record.KeyColumnsSize(); ++i) { - const auto& serializedKey = record.GetKeyColumns(i); - - TSerializedCellVec keyCells; - if (!TSerializedCellVec::TryParse(serializedKey, keyCells)) { - return BadRequest(TStringBuilder() << "Cannot parse key" - << ": serialized# " << serializedKey); - } - - for (const auto& [tableId, info] : TableInfos) { - TVector<TCell> cells(Reserve(info.GetKeyMap().size())); + } + + if (Request) { + Handle(Request); + } else { + Become(&TThis::StateAwaitRequest); + } + } + + /// Process request + + STATEFN(StateAwaitRequest) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvDataShard::TEvEraseRowsRequest, Handle); + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + + void Store(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) { + LOG_D("Store TEvDataShard::TEvEraseRowsRequest"); + Request = ev; + } + + void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev) { + LOG_D("Handle TEvDataShard::TEvEraseRowsRequest"); + + const auto& record = ev->Get()->Record; + + switch (record.GetConditionCase()) { + case NKikimrTxDataShard::TEvEraseRowsRequest::kExpiration: + break; + default: + return BadRequest(TStringBuilder() << "Unknown condition" + << ": condition# " << static_cast<ui32>(record.GetConditionCase())); + } + + THashMap<ui32, ui32> keyColumnIdToIdx; + for (ui32 i = 0; i < record.KeyColumnIdsSize(); ++i) { + Y_VERIFY_DEBUG(!keyColumnIdToIdx.contains(record.GetKeyColumnIds(i))); + keyColumnIdToIdx.emplace(record.GetKeyColumnIds(i), i); + } + + TVector<ui32> indexColumnIds; + { + Y_VERIFY(TableInfos.contains(MainTableId)); + const auto& mainTableInfo = TableInfos.at(MainTableId); + + THashSet<ui32> mainTableKeys; + for (const auto& [_, id] : mainTableInfo.GetKeyMap()) { + Y_VERIFY_DEBUG(!mainTableKeys.contains(id)); + mainTableKeys.insert(id); + } + + for (const auto& id : record.GetKeyColumnIds()) { + if (mainTableKeys.contains(id)) { + continue; + } + + indexColumnIds.push_back(id); + } + } + + THashMap<TTableId, THashMap<ui64, TShardKeys>> keys; // table to shard to keys + TVector<TString> indexColumnValues(Reserve(record.KeyColumnsSize())); + for (ui32 i = 0; i < record.KeyColumnsSize(); ++i) { + const auto& serializedKey = record.GetKeyColumns(i); + + TSerializedCellVec keyCells; + if (!TSerializedCellVec::TryParse(serializedKey, keyCells)) { + return BadRequest(TStringBuilder() << "Cannot parse key" + << ": serialized# " << serializedKey); + } + + for (const auto& [tableId, info] : TableInfos) { + TVector<TCell> cells(Reserve(info.GetKeyMap().size())); for (const auto& [_, id] : info.GetKeyMap()) { - if (!keyColumnIdToIdx.contains(id)) { - return BadRequest(TStringBuilder() << "Key column is absent" - << ": tableId# " << tableId - << ", columnId# " << id); - } - - cells.push_back(keyCells.GetCells()[keyColumnIdToIdx.at(id)]); - } - - const ui64 shardId = GetShardId(TTableRange(cells), info.GetKeyDesc()); - - auto it = keys[tableId].find(shardId); - if (it == keys[tableId].end()) { + if (!keyColumnIdToIdx.contains(id)) { + return BadRequest(TStringBuilder() << "Key column is absent" + << ": tableId# " << tableId + << ", columnId# " << id); + } + + cells.push_back(keyCells.GetCells()[keyColumnIdToIdx.at(id)]); + } + + const ui64 shardId = GetShardId(TTableRange(cells), info.GetKeyDesc()); + + auto it = keys[tableId].find(shardId); + if (it == keys[tableId].end()) { it = keys[tableId].emplace(shardId, TShardKeys()).first; - } - - it->second.Add(TSerializedCellVec::Serialize(cells), i); - - if (tableId != MainTableId) { - continue; - } - - TVector<TCell> indexCells(Reserve(indexColumnIds.size())); - for (const auto& id : indexColumnIds) { - Y_VERIFY(keyColumnIdToIdx.contains(id)); - indexCells.push_back(keyCells.GetCells()[keyColumnIdToIdx.at(id)]); - } - - indexColumnValues.push_back(std::move(TSerializedCellVec::Serialize(indexCells))); - } - } - - Y_VERIFY(keys.contains(MainTableId)); - if (keys.at(MainTableId).size() > 1) { - return ExecError(TStringBuilder() << "Too many main table's shards" - << ": tableId# " << MainTableId - << ", expected# " << 1 - << ", actual# " << keys.at(MainTableId).size()); - } - - auto getDependents = [&keys](const TTableId& mainTableId) { - THashSet<ui64> dependents; - - for (const auto& [tableId, shards] : keys) { - if (tableId != mainTableId) { - for (const auto& [shardId, _] : shards) { - Y_VERIFY_DEBUG(!dependents.contains(shardId)); - dependents.insert(shardId); - } - } - } - - return dependents; - }; - - for (auto& [tableId, data] : keys) { - Y_VERIFY(TableInfos.contains(tableId)); - const auto& keyMap = TableInfos.at(tableId).GetKeyMap(); - - for (auto& kv : data) { - const ui64 shardId = kv.first; - auto& shardKeys = kv.second; - - NKikimrTxDataShard::TDistributedEraseTransaction tx; - - auto& request = *tx.MutableEraseRowsRequest(); - request.SetTableId(tableId.PathId.LocalPathId); - request.SetSchemaVersion(tableId.SchemaVersion); - + } + + it->second.Add(TSerializedCellVec::Serialize(cells), i); + + if (tableId != MainTableId) { + continue; + } + + TVector<TCell> indexCells(Reserve(indexColumnIds.size())); + for (const auto& id : indexColumnIds) { + Y_VERIFY(keyColumnIdToIdx.contains(id)); + indexCells.push_back(keyCells.GetCells()[keyColumnIdToIdx.at(id)]); + } + + indexColumnValues.push_back(std::move(TSerializedCellVec::Serialize(indexCells))); + } + } + + Y_VERIFY(keys.contains(MainTableId)); + if (keys.at(MainTableId).size() > 1) { + return ExecError(TStringBuilder() << "Too many main table's shards" + << ": tableId# " << MainTableId + << ", expected# " << 1 + << ", actual# " << keys.at(MainTableId).size()); + } + + auto getDependents = [&keys](const TTableId& mainTableId) { + THashSet<ui64> dependents; + + for (const auto& [tableId, shards] : keys) { + if (tableId != mainTableId) { + for (const auto& [shardId, _] : shards) { + Y_VERIFY_DEBUG(!dependents.contains(shardId)); + dependents.insert(shardId); + } + } + } + + return dependents; + }; + + for (auto& [tableId, data] : keys) { + Y_VERIFY(TableInfos.contains(tableId)); + const auto& keyMap = TableInfos.at(tableId).GetKeyMap(); + + for (auto& kv : data) { + const ui64 shardId = kv.first; + auto& shardKeys = kv.second; + + NKikimrTxDataShard::TDistributedEraseTransaction tx; + + auto& request = *tx.MutableEraseRowsRequest(); + request.SetTableId(tableId.PathId.LocalPathId); + request.SetSchemaVersion(tableId.SchemaVersion); + for (const auto& [id, _] : keyMap) { - request.AddKeyColumnIds(id); - } - - Y_VERIFY_S(shardKeys.GetKeys().size() == shardKeys.GetPresentRows().Count(), "Rows count mismatch" - << ": expected# " << shardKeys.GetKeys().size() - << ", actual# " << shardKeys.GetPresentRows().Count()); - - for (TString& key : shardKeys.GetKeys()) { - request.AddKeyColumns(std::move(key)); - } - - switch (record.GetConditionCase()) { - case NKikimrTxDataShard::TEvEraseRowsRequest::kExpiration: - request.MutableExpiration()->CopyFrom(record.GetExpiration()); - break; - default: - Y_FAIL_S("Unknown condition: " << static_cast<ui32>(record.GetConditionCase())); - } - - if (tableId == MainTableId) { - for (const ui64 shardId : getDependents(MainTableId)) { - auto& dependents = *tx.AddDependents(); - dependents.SetShardId(shardId); - } - - for (const auto& id : indexColumnIds) { - tx.AddIndexColumnIds(id); - } - - for (auto& value : indexColumnValues) { - tx.AddIndexColumns(std::move(value)); - } - } else { - Y_VERIFY(keys.contains(MainTableId)); - - auto& dependency = *tx.AddDependencies(); - dependency.SetShardId(keys.at(MainTableId).begin()->first); - dependency.SetPresentRows(SerializeBitMap(shardKeys.GetPresentRows())); - } - - LOG_D("Propose tx" - << ": txId# " << TxId - << ", shard# " << shardId - << ", keys# " << request.KeyColumnsSize() - << ", dependents# " << tx.DependentsSize() - << ", dependencies# " << tx.DependenciesSize()); - - auto propose = MakeHolder<TEvDataShard::TEvProposeTransaction>( - NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE, SelfId(), TxId, tx.SerializeAsString() - ); - + request.AddKeyColumnIds(id); + } + + Y_VERIFY_S(shardKeys.GetKeys().size() == shardKeys.GetPresentRows().Count(), "Rows count mismatch" + << ": expected# " << shardKeys.GetKeys().size() + << ", actual# " << shardKeys.GetPresentRows().Count()); + + for (TString& key : shardKeys.GetKeys()) { + request.AddKeyColumns(std::move(key)); + } + + switch (record.GetConditionCase()) { + case NKikimrTxDataShard::TEvEraseRowsRequest::kExpiration: + request.MutableExpiration()->CopyFrom(record.GetExpiration()); + break; + default: + Y_FAIL_S("Unknown condition: " << static_cast<ui32>(record.GetConditionCase())); + } + + if (tableId == MainTableId) { + for (const ui64 shardId : getDependents(MainTableId)) { + auto& dependents = *tx.AddDependents(); + dependents.SetShardId(shardId); + } + + for (const auto& id : indexColumnIds) { + tx.AddIndexColumnIds(id); + } + + for (auto& value : indexColumnValues) { + tx.AddIndexColumns(std::move(value)); + } + } else { + Y_VERIFY(keys.contains(MainTableId)); + + auto& dependency = *tx.AddDependencies(); + dependency.SetShardId(keys.at(MainTableId).begin()->first); + dependency.SetPresentRows(SerializeBitMap(shardKeys.GetPresentRows())); + } + + LOG_D("Propose tx" + << ": txId# " << TxId + << ", shard# " << shardId + << ", keys# " << request.KeyColumnsSize() + << ", dependents# " << tx.DependentsSize() + << ", dependencies# " << tx.DependenciesSize()); + + auto propose = MakeHolder<TEvDataShard::TEvProposeTransaction>( + NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE, SelfId(), TxId, tx.SerializeAsString() + ); + Send(LeaderPipeCache, new TEvPipeCache::TEvForward(propose.Release(), shardId, true)); - - Y_VERIFY_DEBUG(!Shards.contains(shardId)); - Shards.insert(shardId); - - Y_VERIFY_DEBUG(!PendingPrepare.contains(shardId)); - PendingPrepare.insert(shardId); - } - } - - Become(&TThis::StatePropose); - } - - /// Propose - - STATEFN(StatePropose) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePropose); - hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePropose); - default: - return StateExecuteBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandlePropose(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) { - const auto* msg = ev->Get(); - - const ui64 shardId = msg->GetOrigin(); - if (!PendingPrepare.contains(shardId)) { - return; - } - - const auto status = msg->GetStatus(); - LOG_D("HandlePropose TEvDataShard::TEvProposeTransactionResult" - << ": txId# " << TxId - << ", shard# " << shardId - << ", status# " << static_cast<ui32>(status)); - - auto error = [&](TEvResponse::EStatus code, const TStringBuf header) { - return Reply(code, TStringBuilder() << header - << ": reason# " << msg->GetError() - << ", txId# " << TxId - << ", shard# " << shardId); - }; - - switch (status) { - case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: { - Mon->TxResultPrepared->Inc(); - - const auto& record = msg->Record; - - AggrMinStep = Max(AggrMinStep, record.GetMinStep()); - AggrMaxStep = Min(AggrMaxStep, record.GetMaxStep()); - - const auto& domainCoordinators = record.GetDomainCoordinators(); - const auto privateCoordinators = TVector<ui64>(domainCoordinators.begin(), domainCoordinators.end()); - const ui64 privateCoordinator = TCoordinators(privateCoordinators).Select(TxId); - - if (!SelectedCoordinator) { - SelectedCoordinator = privateCoordinator; - } - - if (!SelectedCoordinator || SelectedCoordinator != privateCoordinator) { - CancelProposal(); - Mon->TxResultAborted->Inc(); - return CoordinatorUnknown(TStringBuilder() << "Unable to choose coordinator" - << ": txId# " << TxId - << ", shard# " << shardId - << ", selectCoordinator# " << SelectedCoordinator - << ", privateCoordinator# " << privateCoordinator); - } - - PendingPrepare.erase(shardId); - if (PendingPrepare) { - return; - } - - return RegisterPlan(); - } - case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: - CancelProposal(); - Mon->TxResultComplete->Inc(); - return ExecError(TStringBuilder() << "Unexpected COMPLETE result" - << ": txId# " << TxId - << ", shard# " << shardId); - case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: - CancelProposal(shardId); - Mon->TxResultError->Inc(); + + Y_VERIFY_DEBUG(!Shards.contains(shardId)); + Shards.insert(shardId); + + Y_VERIFY_DEBUG(!PendingPrepare.contains(shardId)); + PendingPrepare.insert(shardId); + } + } + + Become(&TThis::StatePropose); + } + + /// Propose + + STATEFN(StatePropose) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePropose); + hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePropose); + default: + return StateExecuteBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandlePropose(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) { + const auto* msg = ev->Get(); + + const ui64 shardId = msg->GetOrigin(); + if (!PendingPrepare.contains(shardId)) { + return; + } + + const auto status = msg->GetStatus(); + LOG_D("HandlePropose TEvDataShard::TEvProposeTransactionResult" + << ": txId# " << TxId + << ", shard# " << shardId + << ", status# " << static_cast<ui32>(status)); + + auto error = [&](TEvResponse::EStatus code, const TStringBuf header) { + return Reply(code, TStringBuilder() << header + << ": reason# " << msg->GetError() + << ", txId# " << TxId + << ", shard# " << shardId); + }; + + switch (status) { + case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: { + Mon->TxResultPrepared->Inc(); + + const auto& record = msg->Record; + + AggrMinStep = Max(AggrMinStep, record.GetMinStep()); + AggrMaxStep = Min(AggrMaxStep, record.GetMaxStep()); + + const auto& domainCoordinators = record.GetDomainCoordinators(); + const auto privateCoordinators = TVector<ui64>(domainCoordinators.begin(), domainCoordinators.end()); + const ui64 privateCoordinator = TCoordinators(privateCoordinators).Select(TxId); + + if (!SelectedCoordinator) { + SelectedCoordinator = privateCoordinator; + } + + if (!SelectedCoordinator || SelectedCoordinator != privateCoordinator) { + CancelProposal(); + Mon->TxResultAborted->Inc(); + return CoordinatorUnknown(TStringBuilder() << "Unable to choose coordinator" + << ": txId# " << TxId + << ", shard# " << shardId + << ", selectCoordinator# " << SelectedCoordinator + << ", privateCoordinator# " << privateCoordinator); + } + + PendingPrepare.erase(shardId); + if (PendingPrepare) { + return; + } + + return RegisterPlan(); + } + case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: + CancelProposal(); + Mon->TxResultComplete->Inc(); + return ExecError(TStringBuilder() << "Unexpected COMPLETE result" + << ": txId# " << TxId + << ", shard# " << shardId); + case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: + CancelProposal(shardId); + Mon->TxResultError->Inc(); return error(TEvResponse::SHARD_NOT_AVAILABLE, TStringBuf("Not available")); - case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: - Mon->TxResultAborted->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: + Mon->TxResultAborted->Inc(); return error(TEvResponse::SHARD_ABORTED, TStringBuf("Aborted")); - case NKikimrTxDataShard::TEvProposeTransactionResult::TRY_LATER: - CancelProposal(shardId); - Mon->TxResultShardTryLater->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::TRY_LATER: + CancelProposal(shardId); + Mon->TxResultShardTryLater->Inc(); return error(TEvResponse::SHARD_TRY_LATER, TStringBuf("Try later")); - case NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED: - CancelProposal(shardId); - Mon->TxResultShardOverloaded->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED: + CancelProposal(shardId); + Mon->TxResultShardOverloaded->Inc(); return error(TEvResponse::SHARD_OVERLOADED, TStringBuf("Overloaded")); - case NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR: - Mon->TxResultExecError->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR: + Mon->TxResultExecError->Inc(); return error(TEvResponse::SHARD_EXEC_ERROR, TStringBuf("Execution error")); - case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: - Mon->TxResultResultUnavailable->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: + Mon->TxResultResultUnavailable->Inc(); return error(TEvResponse::SHARD_RESULT_UNAVAILABLE, TStringBuf("Result unavailable")); - case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: - Mon->TxResultCancelled->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: + Mon->TxResultCancelled->Inc(); return error(TEvResponse::SHARD_CANCELLED, TStringBuf("Cancelled")); - case NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST: - Mon->TxResultCancelled->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST: + Mon->TxResultCancelled->Inc(); return error(TEvResponse::SHARD_CANCELLED, TStringBuf("Bad request")); - default: - CancelProposal(); - Mon->TxResultFatal->Inc(); - return ShardUnknown(TStringBuilder() << "Unexpected status" - << ": reason# " << msg->GetError() - << ", txId# " << TxId - << ", shard# " << shardId); - } - } - - void HandlePropose(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - const auto* msg = ev->Get(); - - const ui64 shardId = msg->TabletId; - if (!PendingPrepare.contains(shardId)) { - return; - } - - CancelProposal(shardId); - Mon->ClientConnectedError->Inc(); - - if (msg->NotDelivered) { - return Reply(TEvResponse::SHARD_NOT_AVAILABLE, TStringBuilder() << "Could not deliver program" - << ": txId# " << TxId - << ", shard# " << shardId); - } else { - return ShardUnknown(TStringBuilder() << "Tx state unknown" - << ": reason# " << "lost pipe while waiting for reply (propose)" - << ", txId# " << TxId - << ", shard# " << shardId); - } - } - - /// Plan - - void RegisterPlan() { - Y_VERIFY(SelectedCoordinator); - - LOG_D("Register plan" - << ": txId# " << TxId - << ", minStep# " << AggrMinStep - << ", maxStep# " << AggrMaxStep); - - auto propose = MakeHolder<TEvTxProxy::TEvProposeTransaction>( - SelectedCoordinator, TxId, 0, AggrMinStep, AggrMaxStep); - - auto& affectedSet = *propose->Record.MutableTransaction()->MutableAffectedSet(); - affectedSet.Reserve(Shards.size()); - - for (const ui64 shardId : Shards) { - auto& x = *affectedSet.Add(); - x.SetTabletId(shardId); - x.SetFlags(1 << 1 /* AffectedWrite */); - - Y_VERIFY_DEBUG(!PendingResult.contains(shardId)); - PendingResult.insert(shardId); - } - + default: + CancelProposal(); + Mon->TxResultFatal->Inc(); + return ShardUnknown(TStringBuilder() << "Unexpected status" + << ": reason# " << msg->GetError() + << ", txId# " << TxId + << ", shard# " << shardId); + } + } + + void HandlePropose(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + const auto* msg = ev->Get(); + + const ui64 shardId = msg->TabletId; + if (!PendingPrepare.contains(shardId)) { + return; + } + + CancelProposal(shardId); + Mon->ClientConnectedError->Inc(); + + if (msg->NotDelivered) { + return Reply(TEvResponse::SHARD_NOT_AVAILABLE, TStringBuilder() << "Could not deliver program" + << ": txId# " << TxId + << ", shard# " << shardId); + } else { + return ShardUnknown(TStringBuilder() << "Tx state unknown" + << ": reason# " << "lost pipe while waiting for reply (propose)" + << ", txId# " << TxId + << ", shard# " << shardId); + } + } + + /// Plan + + void RegisterPlan() { + Y_VERIFY(SelectedCoordinator); + + LOG_D("Register plan" + << ": txId# " << TxId + << ", minStep# " << AggrMinStep + << ", maxStep# " << AggrMaxStep); + + auto propose = MakeHolder<TEvTxProxy::TEvProposeTransaction>( + SelectedCoordinator, TxId, 0, AggrMinStep, AggrMaxStep); + + auto& affectedSet = *propose->Record.MutableTransaction()->MutableAffectedSet(); + affectedSet.Reserve(Shards.size()); + + for (const ui64 shardId : Shards) { + auto& x = *affectedSet.Add(); + x.SetTabletId(shardId); + x.SetFlags(1 << 1 /* AffectedWrite */); + + Y_VERIFY_DEBUG(!PendingResult.contains(shardId)); + PendingResult.insert(shardId); + } + Send(LeaderPipeCache, new TEvPipeCache::TEvForward(propose.Release(), SelectedCoordinator, true)); - Become(&TThis::StatePlan); - } - - STATEFN(StatePlan) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxy::TEvProposeTransactionStatus, HandlePlan); - hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePlan); - hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePlan); - default: - return StateExecuteBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void HandlePlan(TEvTxProxy::TEvProposeTransactionStatus::TPtr& ev) { - const auto status = ev->Get()->GetStatus(); - LOG_D("Handle TEvTxProxy::TEvProposeTransactionStatus" - << ": txId# " << TxId - << ", status# " << static_cast<ui32>(status)); - - switch (status) { - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted: - Mon->ClientTxStatusAccepted->Inc(); - break; - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusProcessed: - Mon->ClientTxStatusProcessed->Inc(); - break; - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusConfirmed: - Mon->ClientTxStatusConfirmed->Inc(); - break; - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned: - Mon->ClientTxStatusPlanned->Inc(); - break; - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusOutdated: - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclined: - case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclinedNoSpace: + Become(&TThis::StatePlan); + } + + STATEFN(StatePlan) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxy::TEvProposeTransactionStatus, HandlePlan); + hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePlan); + hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePlan); + default: + return StateExecuteBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void HandlePlan(TEvTxProxy::TEvProposeTransactionStatus::TPtr& ev) { + const auto status = ev->Get()->GetStatus(); + LOG_D("Handle TEvTxProxy::TEvProposeTransactionStatus" + << ": txId# " << TxId + << ", status# " << static_cast<ui32>(status)); + + switch (status) { + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted: + Mon->ClientTxStatusAccepted->Inc(); + break; + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusProcessed: + Mon->ClientTxStatusProcessed->Inc(); + break; + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusConfirmed: + Mon->ClientTxStatusConfirmed->Inc(); + break; + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned: + Mon->ClientTxStatusPlanned->Inc(); + break; + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusOutdated: + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclined: + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclinedNoSpace: case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusRestarting: // TODO: retry - CancelProposal(); + CancelProposal(); [[fallthrough]]; - default: - Mon->ClientTxStatusCoordinatorDeclined->Inc(); - return CoordinatorDeclined(TStringBuilder() << "Tx failed to plan" - << ": reason# " << "declined by coodinator" - << ", txId# " << TxId - << ", status# " << static_cast<ui32>(status)); - } - } - - void HandlePlan(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) { - const auto* msg = ev->Get(); - - const ui64 shardId = msg->GetOrigin(); - if (!PendingResult.contains(shardId)) { - return; - } - - const auto status = msg->GetStatus(); - LOG_D("HandlePlan TEvDataShard::TEvProposeTransactionResult" - << ": txId# " << TxId - << ", shard# " << shardId - << ", status# " << static_cast<ui32>(status)); - - auto error = [&](TEvResponse::EStatus code, const TStringBuf header) { - return Reply(code, TStringBuilder() << header - << ": reason# " << msg->GetError() - << ", txId# " << TxId - << ", shard# " << shardId); - }; - - switch (status) { - case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: { - Mon->PlanClientTxResultComplete->Inc(); - - PendingResult.erase(shardId); - if (PendingResult) { - return; - } - - return Reply(); - } - case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: - Mon->PlanClientTxResultAborted->Inc(); + default: + Mon->ClientTxStatusCoordinatorDeclined->Inc(); + return CoordinatorDeclined(TStringBuilder() << "Tx failed to plan" + << ": reason# " << "declined by coodinator" + << ", txId# " << TxId + << ", status# " << static_cast<ui32>(status)); + } + } + + void HandlePlan(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) { + const auto* msg = ev->Get(); + + const ui64 shardId = msg->GetOrigin(); + if (!PendingResult.contains(shardId)) { + return; + } + + const auto status = msg->GetStatus(); + LOG_D("HandlePlan TEvDataShard::TEvProposeTransactionResult" + << ": txId# " << TxId + << ", shard# " << shardId + << ", status# " << static_cast<ui32>(status)); + + auto error = [&](TEvResponse::EStatus code, const TStringBuf header) { + return Reply(code, TStringBuilder() << header + << ": reason# " << msg->GetError() + << ", txId# " << TxId + << ", shard# " << shardId); + }; + + switch (status) { + case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: { + Mon->PlanClientTxResultComplete->Inc(); + + PendingResult.erase(shardId); + if (PendingResult) { + return; + } + + return Reply(); + } + case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: + Mon->PlanClientTxResultAborted->Inc(); return error(TEvResponse::SHARD_ABORTED, TStringBuf("Aborted")); - case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: - Mon->PlanClientTxResultResultUnavailable->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: + Mon->PlanClientTxResultResultUnavailable->Inc(); return error(TEvResponse::SHARD_RESULT_UNAVAILABLE, TStringBuf("Result unavailable")); - case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: - Mon->PlanClientTxResultCancelled->Inc(); + case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: + Mon->PlanClientTxResultCancelled->Inc(); return error(TEvResponse::SHARD_CANCELLED, TStringBuf("Cancelled")); - default: - Mon->PlanClientTxResultExecError->Inc(); + default: + Mon->PlanClientTxResultExecError->Inc(); return error(TEvResponse::SHARD_EXEC_ERROR, TStringBuf("Unexpected status")); - } - } - - void HandlePlan(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - const auto* msg = ev->Get(); - - if (SelectedCoordinator == msg->TabletId) { - if (msg->NotDelivered) { - Mon->PlanCoordinatorDeclined->Inc(); - return CoordinatorDeclined(TStringBuilder() << "Tx failed to plan" - << ": reason# " << "not delivered to coordinator" - << ", txId# " << TxId - << ", coordinator# " << msg->TabletId); - } else { - Mon->PlanClientDestroyed->Inc(); - return CoordinatorUnknown(TStringBuilder() << "Tx state unknown" - << ": reason# " << "delivery problem to coordinator" - << ", txId# " << TxId - << ", coordinator# " << msg->TabletId); - } - } - - if (PendingResult.contains(msg->TabletId)) { - Mon->ClientConnectedError->Inc(); - return ShardUnknown(TStringBuilder() << "Tx state unknown" - << ": reason# " << "lost pipe while waiting for reply (plan)" - << ", txId# " << TxId - << ", shard# " << msg->TabletId); - } - } - - void PassAway() override { - Cancelled = true; - - if (ResolvingKeys) { - return; - } - - IActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::DISTRIBUTED_ERASE_ROWS_ACTOR; - } - - TDistEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes) - : ReplyTo(replyTo) - , MainTableId(mainTableId) - , Indexes(indexes) - , Cancelled(false) - , ResolvingKeys(false) - , TxId(0) - , SelectedCoordinator(0) - , AggrMinStep(0) - , AggrMaxStep(Max<ui64>()) - { - } - - void Bootstrap() { - AllocateTxId(); - } - - STATEFN(StatePrepareBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvDataShard::TEvEraseRowsRequest, Store); - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - - STATEFN(StateExecuteBase) { - switch (ev->GetTypeRewrite()) { - cFunc(TEvents::TEvUndelivered::EventType, DeliveryProblem); - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - -private: - const TActorId ReplyTo; - const TTableId MainTableId; - const TIndexes Indexes; - - bool Cancelled; - bool ResolvingKeys; - - ui64 TxId; + } + } + + void HandlePlan(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + const auto* msg = ev->Get(); + + if (SelectedCoordinator == msg->TabletId) { + if (msg->NotDelivered) { + Mon->PlanCoordinatorDeclined->Inc(); + return CoordinatorDeclined(TStringBuilder() << "Tx failed to plan" + << ": reason# " << "not delivered to coordinator" + << ", txId# " << TxId + << ", coordinator# " << msg->TabletId); + } else { + Mon->PlanClientDestroyed->Inc(); + return CoordinatorUnknown(TStringBuilder() << "Tx state unknown" + << ": reason# " << "delivery problem to coordinator" + << ", txId# " << TxId + << ", coordinator# " << msg->TabletId); + } + } + + if (PendingResult.contains(msg->TabletId)) { + Mon->ClientConnectedError->Inc(); + return ShardUnknown(TStringBuilder() << "Tx state unknown" + << ": reason# " << "lost pipe while waiting for reply (plan)" + << ", txId# " << TxId + << ", shard# " << msg->TabletId); + } + } + + void PassAway() override { + Cancelled = true; + + if (ResolvingKeys) { + return; + } + + IActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::DISTRIBUTED_ERASE_ROWS_ACTOR; + } + + TDistEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes) + : ReplyTo(replyTo) + , MainTableId(mainTableId) + , Indexes(indexes) + , Cancelled(false) + , ResolvingKeys(false) + , TxId(0) + , SelectedCoordinator(0) + , AggrMinStep(0) + , AggrMaxStep(Max<ui64>()) + { + } + + void Bootstrap() { + AllocateTxId(); + } + + STATEFN(StatePrepareBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvDataShard::TEvEraseRowsRequest, Store); + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + + STATEFN(StateExecuteBase) { + switch (ev->GetTypeRewrite()) { + cFunc(TEvents::TEvUndelivered::EventType, DeliveryProblem); + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + +private: + const TActorId ReplyTo; + const TTableId MainTableId; + const TIndexes Indexes; + + bool Cancelled; + bool ResolvingKeys; + + ui64 TxId; TActorId LeaderPipeCache; - TIntrusivePtr<NTxProxy::TTxProxyMon> Mon; - - THashMap<TTableId, TTableInfo> TableInfos; - TEvDataShard::TEvEraseRowsRequest::TPtr Request; - - THashSet<ui64> Shards; - THashSet<ui64> PendingPrepare; - THashSet<ui64> PendingResult; - - ui64 SelectedCoordinator; - ui64 AggrMinStep; - ui64 AggrMaxStep; - -}; // TDistEraser - -IActor* CreateDistributedEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes) { - return new TDistEraser(replyTo, mainTableId, indexes); -} - + TIntrusivePtr<NTxProxy::TTxProxyMon> Mon; + + THashMap<TTableId, TTableInfo> TableInfos; + TEvDataShard::TEvEraseRowsRequest::TPtr Request; + + THashSet<ui64> Shards; + THashSet<ui64> PendingPrepare; + THashSet<ui64> PendingResult; + + ui64 SelectedCoordinator; + ui64 AggrMinStep; + ui64 AggrMaxStep; + +}; // TDistEraser + +IActor* CreateDistributedEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes) { + return new TDistEraser(replyTo, mainTableId, indexes); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_distributed_erase.h b/ydb/core/tx/datashard/datashard_distributed_erase.h index 55b5b4d8d1a..c20c3ed0a98 100644 --- a/ydb/core/tx/datashard/datashard_distributed_erase.h +++ b/ydb/core/tx/datashard/datashard_distributed_erase.h @@ -1,36 +1,36 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/scheme/scheme_tabledefs.h> - -#include <util/generic/hash.h> -#include <util/generic/vector.h> -#include <util/stream/str.h> - -namespace NKikimr { + +#include <util/generic/hash.h> +#include <util/generic/vector.h> +#include <util/stream/str.h> + +namespace NKikimr { namespace NDataShard { - -using TKeyMap = TVector<std::pair<ui32, ui32>>; // index to main -using TIndexes = THashMap<TTableId, TKeyMap>; - -IActor* CreateDistributedEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes); - -template <typename TBitMapType> -static TString SerializeBitMap(const TBitMapType& bitmap) { - TString serialized; - TStringOutput output(serialized); - bitmap.Save(&output); - return serialized; -} - -template <typename TBitMapType> -static TBitMapType DeserializeBitMap(const TString& serialized) { - TStringInput input(serialized); - TBitMapType bitmap; - bitmap.Load(&input); - return bitmap; -} - + +using TKeyMap = TVector<std::pair<ui32, ui32>>; // index to main +using TIndexes = THashMap<TTableId, TKeyMap>; + +IActor* CreateDistributedEraser(const TActorId& replyTo, const TTableId& mainTableId, const TIndexes& indexes); + +template <typename TBitMapType> +static TString SerializeBitMap(const TBitMapType& bitmap) { + TString serialized; + TStringOutput output(serialized); + bitmap.Save(&output); + return serialized; +} + +template <typename TBitMapType> +static TBitMapType DeserializeBitMap(const TString& serialized) { + TStringInput input(serialized); + TBitMapType bitmap; + bitmap.Load(&input); + return bitmap; +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 6d6728a249a..11313474f80 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -5,17 +5,17 @@ #include "datashard_trans_queue.h" #include "datashard_outreadset.h" #include "datashard_pipeline.h" -#include "datashard_schema_snapshots.h" +#include "datashard_schema_snapshots.h" #include "datashard_snapshots.h" -#include "datashard_s3_downloads.h" -#include "datashard_s3_uploads.h" +#include "datashard_s3_downloads.h" +#include "datashard_s3_uploads.h" #include "datashard_user_table.h" #include "datashard_build_index.h" #include "datashard_repl_offsets.h" #include "datashard_repl_offsets_client.h" #include "datashard_repl_offsets_server.h" -#include "change_exchange.h" -#include "change_record.h" +#include "change_exchange.h" +#include "change_record.h" #include "progress_queue.h" #include "read_iterator.h" @@ -45,8 +45,8 @@ #include <library/cpp/actors/interconnect/interconnect.h> -#include <util/string/join.h> - +#include <util/string/join.h> + namespace NKikimr { namespace NDataShard { @@ -188,22 +188,22 @@ class TDataShard class TTxDiscardVolatileSnapshot; class TTxCleanupRemovedSnapshots; class TTxMigrateSchemeShard; - class TTxGetS3Upload; - class TTxStoreS3UploadId; - class TTxChangeS3UploadStatus; - class TTxGetS3DownloadInfo; - class TTxStoreS3DownloadInfo; - class TTxUnsafeUploadRows; + class TTxGetS3Upload; + class TTxStoreS3UploadId; + class TTxChangeS3UploadStatus; + class TTxGetS3DownloadInfo; + class TTxStoreS3DownloadInfo; + class TTxUnsafeUploadRows; class TTxExecuteMvccStateChange; class TTxGetRemovedRowVersions; class TTxCompactBorrowed; class TTxCompactTable; class TTxPersistFullCompactionTs; - template <typename T> friend class TTxDirectBase; - class TTxUploadRows; - class TTxEraseRows; - + template <typename T> friend class TTxDirectBase; + class TTxUploadRows; + class TTxEraseRows; + ITransaction *CreateTxMonitoring(TDataShard *self, NMon::TEvRemoteHttpInfo::TPtr ev); ITransaction *CreateTxGetInfo(TDataShard *self, @@ -240,7 +240,7 @@ class TDataShard friend class TEngineBay; friend class NMiniKQL::TKqpScanComputeContext; friend class TSnapshotManager; - friend class TSchemaSnapshotManager; + friend class TSchemaSnapshotManager; friend class TReplicationSourceOffsetsClient; friend class TReplicationSourceOffsetsServer; @@ -249,12 +249,12 @@ class TDataShard friend class TWaitForStreamClearanceUnit; friend class TBuildIndexScan; friend class TReadColumnsScan; - friend class TCondEraseScan; + friend class TCondEraseScan; friend class TDatashardKeySampler; - friend class TS3UploadsManager; - friend class TS3DownloadsManager; - friend class TS3Downloader; + friend class TS3UploadsManager; + friend class TS3DownloadsManager; + friend class TS3Downloader; friend struct TSetupSysLocks; friend class TTxStartMvccStateChange; @@ -264,14 +264,14 @@ class TDataShard class TTxPersistSubDomainPathId; class TTxPersistSubDomainOutOfSpace; - class TTxRequestChangeRecords; - class TTxRemoveChangeRecords; - class TTxChangeExchangeHandshake; - class TTxApplyChangeRecords; - class TTxActivateChangeSender; - class TTxActivateChangeSenderAck; - class TTxChangeExchangeSplitAck; - + class TTxRequestChangeRecords; + class TTxRemoveChangeRecords; + class TTxChangeExchangeHandshake; + class TTxApplyChangeRecords; + class TTxActivateChangeSender; + class TTxActivateChangeSenderAck; + class TTxChangeExchangeSplitAck; + class TTxApplyReplicationChanges; struct TEvPrivate { @@ -291,11 +291,11 @@ class TDataShard EvScanStats, EvPersistScanState, EvPersistScanStateAck, - EvConditionalEraseRowsRegistered, - EvAsyncJobComplete, + EvConditionalEraseRowsRegistered, + EvAsyncJobComplete, EvSubDomainPathIdFound, - EvRequestChangeRecords, - EvRemoveChangeRecords, + EvRequestChangeRecords, + EvRemoveChangeRecords, EvReplicationSourceOffsets, EvEnd }; @@ -384,26 +384,26 @@ class TDataShard }; struct TEvPersistScanStateAck : public TEventLocal<TEvPersistScanStateAck, EvPersistScanStateAck> {}; - - struct TEvConditionalEraseRowsRegistered : public TEventLocal<TEvConditionalEraseRowsRegistered, EvConditionalEraseRowsRegistered> { + + struct TEvConditionalEraseRowsRegistered : public TEventLocal<TEvConditionalEraseRowsRegistered, EvConditionalEraseRowsRegistered> { explicit TEvConditionalEraseRowsRegistered(ui64 txId, const TActorId& actorId) - : TxId(txId) - , ActorId(actorId) - { - } - - const ui64 TxId; + : TxId(txId) + , ActorId(actorId) + { + } + + const ui64 TxId; const TActorId ActorId; - }; - - struct TEvAsyncJobComplete : public TEventLocal<TEvAsyncJobComplete, EvAsyncJobComplete> { + }; + + struct TEvAsyncJobComplete : public TEventLocal<TEvAsyncJobComplete, EvAsyncJobComplete> { explicit TEvAsyncJobComplete(TAutoPtr<IDestructable> prod) - : Prod(prod) - { - } - + : Prod(prod) + { + } + TAutoPtr<IDestructable> Prod; - }; + }; struct TEvSubDomainPathIdFound : public TEventLocal<TEvSubDomainPathIdFound, EvSubDomainPathIdFound> { TEvSubDomainPathIdFound(ui64 schemeShardId, ui64 localPathId) @@ -414,9 +414,9 @@ class TDataShard const ui64 SchemeShardId; const ui64 LocalPathId; }; - - struct TEvRequestChangeRecords : public TEventLocal<TEvRequestChangeRecords, EvRequestChangeRecords> {}; - struct TEvRemoveChangeRecords : public TEventLocal<TEvRemoveChangeRecords, EvRemoveChangeRecords> {}; + + struct TEvRequestChangeRecords : public TEventLocal<TEvRequestChangeRecords, EvRequestChangeRecords> {}; + struct TEvRemoveChangeRecords : public TEventLocal<TEvRemoveChangeRecords, EvRemoveChangeRecords> {}; struct TEvReplicationSourceOffsets : public TEventLocal<TEvReplicationSourceOffsets, EvReplicationSourceOffsets> { struct TSplitKey { @@ -540,14 +540,14 @@ class TDataShard struct PlanStep : Column<7, NScheme::NTypeIds::Uint64> {}; struct ReadOnly : Column<8, NScheme::NTypeIds::Bool> {}; - struct Success : Column<9, NScheme::NTypeIds::Bool> {}; - struct Error : Column<10, NScheme::NTypeIds::String> { using Type = TString; }; - struct DataSize : Column<11, NScheme::NTypeIds::Uint64> {}; // Bytes - struct Rows : Column<12, NScheme::NTypeIds::Uint64> {}; - + struct Success : Column<9, NScheme::NTypeIds::Bool> {}; + struct Error : Column<10, NScheme::NTypeIds::String> { using Type = TString; }; + struct DataSize : Column<11, NScheme::NTypeIds::Uint64> {}; // Bytes + struct Rows : Column<12, NScheme::NTypeIds::Uint64> {}; + using TKey = TableKey<TxId>; using TColumns = TableColumns<TxId, Operation, Source, SourceTablet, - MinStep, MaxStep, PlanStep, ReadOnly, Success, Error, DataSize, Rows>; + MinStep, MaxStep, PlanStep, ReadOnly, Success, Error, DataSize, Rows>; }; // Here we persist snapshots metadata to preserve it across Src datashard restarts @@ -591,8 +591,8 @@ class TDataShard }; struct Snapshots : Table<14> { - struct Oid : Column<1, NScheme::NTypeIds::Uint64> {}; // PathOwnerId - struct Tid : Column<2, NScheme::NTypeIds::Uint64> {}; // LocalPathId + struct Oid : Column<1, NScheme::NTypeIds::Uint64> {}; // PathOwnerId + struct Tid : Column<2, NScheme::NTypeIds::Uint64> {}; // LocalPathId struct Step : Column<3, NScheme::NTypeIds::Uint64> {}; struct TxId : Column<4, NScheme::NTypeIds::Uint64> {}; struct Name : Column<5, NScheme::NTypeIds::String> {}; @@ -603,97 +603,97 @@ class TDataShard using TColumns = TableColumns<Oid, Tid, Step, TxId, Name, Flags, TimeoutMs>; }; - struct S3Uploads : Table<15> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct UploadId : Column<2, NScheme::NTypeIds::String> { using Type = TString; }; - struct Status : Column<3, NScheme::NTypeIds::Uint8> { using Type = TS3Upload::EStatus; static constexpr Type Default = TS3Upload::EStatus::UploadParts; }; - struct Error : Column<4, NScheme::NTypeIds::Utf8> { using Type = TString; }; - - using TKey = TableKey<TxId>; - using TColumns = TableColumns<TxId, UploadId, Status, Error>; - }; - - // deprecated - struct S3UploadParts : Table<20> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct PartNumber : Column<2, NScheme::NTypeIds::Uint32> {}; - struct ETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; - - using TKey = TableKey<TxId>; - using TColumns = TableColumns<TxId, PartNumber, ETag>; - }; - - struct S3UploadedParts : Table<23> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct PartNumber : Column<2, NScheme::NTypeIds::Uint32> {}; - struct ETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; - - using TKey = TableKey<TxId, PartNumber>; - using TColumns = TableColumns<TxId, PartNumber, ETag>; - }; - - struct S3Downloads : Table<16> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct SchemeETag : Column<2, NScheme::NTypeIds::String> { using Type = TString; }; - struct DataETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; - struct ProcessedBytes : Column<4, NScheme::NTypeIds::Uint64> {}; - struct WrittenBytes : Column<5, NScheme::NTypeIds::Uint64> {}; - struct WrittenRows : Column<6, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<TxId>; - using TColumns = TableColumns<TxId, SchemeETag, DataETag, ProcessedBytes, WrittenBytes, WrittenRows>; - }; - - struct ChangeRecords : Table<17> { - struct Order : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Group : Column<2, NScheme::NTypeIds::Uint64> {}; - struct PlanStep : Column<3, NScheme::NTypeIds::Uint64> {}; - struct TxId : Column<4, NScheme::NTypeIds::Uint64> {}; - struct PathOwnerId : Column<5, NScheme::NTypeIds::Uint64> {}; - struct LocalPathId : Column<6, NScheme::NTypeIds::Uint64> {}; - struct BodySize : Column<7, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<Order>; - using TColumns = TableColumns<Order, Group, PlanStep, TxId, PathOwnerId, LocalPathId, BodySize>; - }; - - struct ChangeRecordDetails : Table<18> { - struct Order : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Kind : Column<2, NScheme::NTypeIds::Uint8> { using Type = TChangeRecord::EKind; }; - struct Body : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; - - using TKey = TableKey<Order>; - using TColumns = TableColumns<Order, Kind, Body>; - }; - - struct ChangeSenders : Table<19> { - struct Origin : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Generation : Column<2, NScheme::NTypeIds::Uint64> {}; - struct LastRecordOrder : Column<3, NScheme::NTypeIds::Uint64> {}; - struct LastSeenAt : Column<4, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; - - using TKey = TableKey<Origin>; - using TColumns = TableColumns<Origin, Generation, LastRecordOrder, LastSeenAt>; - }; - - // Table<20> was taken by S3UploadParts - - struct SrcChangeSenderActivations : Table<21> { - struct DstTabletId : Column<1, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<DstTabletId>; - using TColumns = TableColumns<DstTabletId>; - }; - - struct DstChangeSenderActivations : Table<22> { - struct SrcTabletId : Column<1, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<SrcTabletId>; - using TColumns = TableColumns<SrcTabletId>; - }; - - // Table<23> is taken by S3UploadedParts - + struct S3Uploads : Table<15> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct UploadId : Column<2, NScheme::NTypeIds::String> { using Type = TString; }; + struct Status : Column<3, NScheme::NTypeIds::Uint8> { using Type = TS3Upload::EStatus; static constexpr Type Default = TS3Upload::EStatus::UploadParts; }; + struct Error : Column<4, NScheme::NTypeIds::Utf8> { using Type = TString; }; + + using TKey = TableKey<TxId>; + using TColumns = TableColumns<TxId, UploadId, Status, Error>; + }; + + // deprecated + struct S3UploadParts : Table<20> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PartNumber : Column<2, NScheme::NTypeIds::Uint32> {}; + struct ETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; + + using TKey = TableKey<TxId>; + using TColumns = TableColumns<TxId, PartNumber, ETag>; + }; + + struct S3UploadedParts : Table<23> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PartNumber : Column<2, NScheme::NTypeIds::Uint32> {}; + struct ETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; + + using TKey = TableKey<TxId, PartNumber>; + using TColumns = TableColumns<TxId, PartNumber, ETag>; + }; + + struct S3Downloads : Table<16> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct SchemeETag : Column<2, NScheme::NTypeIds::String> { using Type = TString; }; + struct DataETag : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; + struct ProcessedBytes : Column<4, NScheme::NTypeIds::Uint64> {}; + struct WrittenBytes : Column<5, NScheme::NTypeIds::Uint64> {}; + struct WrittenRows : Column<6, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<TxId>; + using TColumns = TableColumns<TxId, SchemeETag, DataETag, ProcessedBytes, WrittenBytes, WrittenRows>; + }; + + struct ChangeRecords : Table<17> { + struct Order : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Group : Column<2, NScheme::NTypeIds::Uint64> {}; + struct PlanStep : Column<3, NScheme::NTypeIds::Uint64> {}; + struct TxId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct PathOwnerId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct LocalPathId : Column<6, NScheme::NTypeIds::Uint64> {}; + struct BodySize : Column<7, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<Order>; + using TColumns = TableColumns<Order, Group, PlanStep, TxId, PathOwnerId, LocalPathId, BodySize>; + }; + + struct ChangeRecordDetails : Table<18> { + struct Order : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Kind : Column<2, NScheme::NTypeIds::Uint8> { using Type = TChangeRecord::EKind; }; + struct Body : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; + + using TKey = TableKey<Order>; + using TColumns = TableColumns<Order, Kind, Body>; + }; + + struct ChangeSenders : Table<19> { + struct Origin : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Generation : Column<2, NScheme::NTypeIds::Uint64> {}; + struct LastRecordOrder : Column<3, NScheme::NTypeIds::Uint64> {}; + struct LastSeenAt : Column<4, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; + + using TKey = TableKey<Origin>; + using TColumns = TableColumns<Origin, Generation, LastRecordOrder, LastSeenAt>; + }; + + // Table<20> was taken by S3UploadParts + + struct SrcChangeSenderActivations : Table<21> { + struct DstTabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<DstTabletId>; + using TColumns = TableColumns<DstTabletId>; + }; + + struct DstChangeSenderActivations : Table<22> { + struct SrcTabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<SrcTabletId>; + using TColumns = TableColumns<SrcTabletId>; + }; + + // Table<23> is taken by S3UploadedParts + struct ReplicationSourceOffsets : Table<24> { struct PathOwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; struct TablePathId : Column<2, NScheme::NTypeIds::Uint64> {}; @@ -736,24 +736,24 @@ class TDataShard using TColumns = TableColumns<Tid, FullCompactionTs>; }; - struct SchemaSnapshots : Table<28> { - struct PathOwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct SchemaVersion : Column<3, NScheme::NTypeIds::Uint64> {}; - struct Step : Column<4, NScheme::NTypeIds::Uint64> {}; - struct TxId : Column<5, NScheme::NTypeIds::Uint64> {}; - struct Schema : Column<6, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<PathOwnerId, LocalPathId, SchemaVersion>; - using TColumns = TableColumns<PathOwnerId, LocalPathId, SchemaVersion, Step, TxId, Schema>; - }; - + struct SchemaSnapshots : Table<28> { + struct PathOwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct SchemaVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + struct Step : Column<4, NScheme::NTypeIds::Uint64> {}; + struct TxId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct Schema : Column<6, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<PathOwnerId, LocalPathId, SchemaVersion>; + using TColumns = TableColumns<PathOwnerId, LocalPathId, SchemaVersion, Step, TxId, Schema>; + }; + using TTables = SchemaTables<Sys, UserTables, TxMain, TxDetails, InReadSets, OutReadSets, PlanQueue, DeadlineQueue, SchemaOperations, SplitSrcSnapshots, SplitDstReceivedSnapshots, TxArtifacts, ScanProgress, - Snapshots, S3Uploads, S3Downloads, ChangeRecords, ChangeRecordDetails, ChangeSenders, S3UploadedParts, + Snapshots, S3Uploads, S3Downloads, ChangeRecords, ChangeRecordDetails, ChangeSenders, S3UploadedParts, SrcChangeSenderActivations, DstChangeSenderActivations, ReplicationSourceOffsets, ReplicationSources, DstReplicationSourceOffsetsReceived, - UserTablesStats, SchemaSnapshots>; + UserTablesStats, SchemaSnapshots>; // These settings are persisted on each Init. So we use empty settings in order not to overwrite what // was changed by the user @@ -802,9 +802,9 @@ class TDataShard Sys_SubDomainLocalPathId, // 34 LocalPathId of the subdomain path id Sys_SubDomainOutOfSpace, // 35 Boolean flag indicating database is out of space - Sys_NextChangeRecordOrder, // 36 Next order of change record - Sys_LastChangeRecordGroup, // 37 Last group number of change records - + Sys_NextChangeRecordOrder, // 36 Next order of change record + Sys_LastChangeRecordGroup, // 37 Last group number of change records + // reserved SysPipeline_Flags = 1000, SysPipeline_LimitActiveTx, @@ -919,9 +919,9 @@ class TDataShard void Handle(TEvDataShard::TEvS3ListingRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvUploadRowsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvConditionalEraseRowsRegistered::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvConditionalEraseRowsRegistered::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvRead::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvReadContinue::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvReadAck::TPtr& ev, const TActorContext& ctx); @@ -939,18 +939,18 @@ class TDataShard void Handle(TEvDataShard::TEvRefreshVolatileSnapshotRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvDiscardVolatileSnapshotRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvMigrateSchemeShardRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvGetS3Upload::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvStoreS3UploadId::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvChangeS3UploadStatus::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvGetS3DownloadInfo::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvStoreS3DownloadInfo::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvUnsafeUploadRowsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvGetS3Upload::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvStoreS3UploadId::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvChangeS3UploadStatus::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvGetS3DownloadInfo::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvStoreS3DownloadInfo::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvUnsafeUploadRowsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvBuildIndexCreateRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvAsyncJobComplete::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvAsyncJobComplete::TPtr& ev, const TActorContext& ctx); + + void Handle(TEvDataShard::TEvCancelBackup::TPtr &ev, const TActorContext &ctx); + void Handle(TEvDataShard::TEvCancelRestore::TPtr &ev, const TActorContext &ctx); - void Handle(TEvDataShard::TEvCancelBackup::TPtr &ev, const TActorContext &ctx); - void Handle(TEvDataShard::TEvCancelRestore::TPtr &ev, const TActorContext &ctx); - void Handle(TEvTxProcessing::TEvStreamClearanceResponse::TPtr &ev, const TActorContext &ctx) { ForwardEventToOperation(ev, ctx); } @@ -968,21 +968,21 @@ class TDataShard void Handle(TEvPrivate::TEvSubDomainPathIdFound::TPtr& ev, const TActorContext& ctx); void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev, const TActorContext& ctx); - // change sending - void Handle(TEvChangeExchange::TEvRequestRecords::TPtr& ev, const TActorContext& ctx); - void Handle(TEvChangeExchange::TEvRemoveRecords::TPtr& ev, const TActorContext& ctx); - void ScheduleRequestChangeRecords(const TActorContext& ctx); - void ScheduleRemoveChangeRecords(const TActorContext& ctx); - void Handle(TEvPrivate::TEvRequestChangeRecords::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvRemoveChangeRecords::TPtr& ev, const TActorContext& ctx); - // change receiving - void Handle(TEvChangeExchange::TEvHandshake::TPtr& ev, const TActorContext& ctx); - void Handle(TEvChangeExchange::TEvApplyRecords::TPtr& ev, const TActorContext& ctx); - // activation - void Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev, const TActorContext& ctx); - void Handle(TEvChangeExchange::TEvActivateSenderAck::TPtr& ev, const TActorContext& ctx); - void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev, const TActorContext& ctx); - + // change sending + void Handle(TEvChangeExchange::TEvRequestRecords::TPtr& ev, const TActorContext& ctx); + void Handle(TEvChangeExchange::TEvRemoveRecords::TPtr& ev, const TActorContext& ctx); + void ScheduleRequestChangeRecords(const TActorContext& ctx); + void ScheduleRemoveChangeRecords(const TActorContext& ctx); + void Handle(TEvPrivate::TEvRequestChangeRecords::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvRemoveChangeRecords::TPtr& ev, const TActorContext& ctx); + // change receiving + void Handle(TEvChangeExchange::TEvHandshake::TPtr& ev, const TActorContext& ctx); + void Handle(TEvChangeExchange::TEvApplyRecords::TPtr& ev, const TActorContext& ctx); + // activation + void Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev, const TActorContext& ctx); + void Handle(TEvChangeExchange::TEvActivateSenderAck::TPtr& ev, const TActorContext& ctx); + void Handle(TEvChangeExchange::TEvSplitAck::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvGetRemovedRowVersions::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvCompactBorrowed::TPtr& ev, const TActorContext& ctx); @@ -1017,7 +1017,7 @@ class TDataShard void OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActorContext &ctx) override; void OnActivateExecutor(const TActorContext &ctx) override; - void Cleanup(const TActorContext &ctx); + void Cleanup(const TActorContext &ctx); void SwitchToWork(const TActorContext &ctx); void SyncConfig(); @@ -1340,38 +1340,38 @@ public: return Executor()->Step(); } - TSchemaOperation *FindSchemaTx(ui64 txId) { return TransQueue.FindSchemaTx(txId); } + TSchemaOperation *FindSchemaTx(ui64 txId) { return TransQueue.FindSchemaTx(txId); } - TUserTable::TPtr AlterTableSchemaVersion( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, const ui64 tableSchemaVersion, - bool persist = true); + TUserTable::TPtr AlterTableSchemaVersion( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, const ui64 tableSchemaVersion, + bool persist = true); - TUserTable::TPtr AlterTableAddIndex( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, + TUserTable::TPtr AlterTableAddIndex( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, const NKikimrSchemeOp::TIndexDescription& indexDesc); - - TUserTable::TPtr AlterTableDropIndex( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& indexPathId); - - TUserTable::TPtr AlterTableAddCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, + + TUserTable::TPtr AlterTableDropIndex( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& indexPathId); + + TUserTable::TPtr AlterTableAddCdcStream( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, const NKikimrSchemeOp::TCdcStreamDescription& streamDesc); - - TUserTable::TPtr AlterTableDisableCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& streamPathId); - - TUserTable::TPtr AlterTableDropCdcStream( - const TActorContext& ctx, TTransactionContext& txc, - const TPathId& pathId, ui64 tableSchemaVersion, - const TPathId& streamPathId); - + + TUserTable::TPtr AlterTableDisableCdcStream( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& streamPathId); + + TUserTable::TPtr AlterTableDropCdcStream( + const TActorContext& ctx, TTransactionContext& txc, + const TPathId& pathId, ui64 tableSchemaVersion, + const TPathId& streamPathId); + TUserTable::TPtr CreateUserTable(TTransactionContext& txc, const NKikimrSchemeOp::TTableDescription& tableScheme); TUserTable::TPtr AlterUserTable(const TActorContext& ctx, TTransactionContext& txc, const NKikimrSchemeOp::TTableDescription& tableScheme); @@ -1381,34 +1381,34 @@ public: ui32 GetLastLocalTid() const { return LastLocalTid; } - ui64 AllocateChangeRecordOrder(NIceDb::TNiceDb& db); - ui64 AllocateChangeRecordGroup(NIceDb::TNiceDb& db); - void PersistChangeRecord(NIceDb::TNiceDb& db, const TChangeRecord& record); + ui64 AllocateChangeRecordOrder(NIceDb::TNiceDb& db); + ui64 AllocateChangeRecordGroup(NIceDb::TNiceDb& db); + void PersistChangeRecord(NIceDb::TNiceDb& db, const TChangeRecord& record); void MoveChangeRecord(NIceDb::TNiceDb& db, ui64 order, const TPathId& pathId); - void RemoveChangeRecord(NIceDb::TNiceDb& db, ui64 order); - void EnqueueChangeRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records); - void CreateChangeSender(const TActorContext& ctx); - void KillChangeSender(const TActorContext& ctx); - void MaybeActivateChangeSender(const TActorContext& ctx); - const TActorId& GetChangeSender() const { return OutChangeSender; } + void RemoveChangeRecord(NIceDb::TNiceDb& db, ui64 order); + void EnqueueChangeRecords(TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>&& records); + void CreateChangeSender(const TActorContext& ctx); + void KillChangeSender(const TActorContext& ctx); + void MaybeActivateChangeSender(const TActorContext& ctx); + const TActorId& GetChangeSender() const { return OutChangeSender; } bool LoadChangeRecords(NIceDb::TNiceDb& db, TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo>& changeRecords); - - static void PersistSchemeTxResult(NIceDb::TNiceDb &db, const TSchemaOperation& op); + + static void PersistSchemeTxResult(NIceDb::TNiceDb &db, const TSchemaOperation& op); void NotifySchemeshard(const TActorContext& ctx, ui64 txId = 0); - + TThrRefBase* GetDataShardSysTables() { return DataShardSysTables.Get(); } TSnapshotManager& GetSnapshotManager() { return SnapshotManager; } const TSnapshotManager& GetSnapshotManager() const { return SnapshotManager; } - TSchemaSnapshotManager& GetSchemaSnapshotManager() { return SchemaSnapshotManager; } - const TSchemaSnapshotManager& GetSchemaSnapshotManager() const { return SchemaSnapshotManager; } - - template <typename... Args> - bool PromoteCompleteEdge(Args&&... args) { - return SnapshotManager.PromoteCompleteEdge(std::forward<Args>(args)...); - } + TSchemaSnapshotManager& GetSchemaSnapshotManager() { return SchemaSnapshotManager; } + const TSchemaSnapshotManager& GetSchemaSnapshotManager() const { return SchemaSnapshotManager; } + + template <typename... Args> + bool PromoteCompleteEdge(Args&&... args) { + return SnapshotManager.PromoteCompleteEdge(std::forward<Args>(args)...); + } TBuildIndexManager& GetBuildIndexManager() { return BuildIndexManager; } const TBuildIndexManager& GetBuildIndexManager() const { return BuildIndexManager; } @@ -1441,8 +1441,8 @@ public: bool WaitPlanStep(ui64 step); bool CheckTxNeedWait(const TEvDataShard::TEvProposeTransaction::TPtr& ev) const; - bool CheckChangesQueueOverflow() const; - + bool CheckChangesQueueOverflow() const; + private: /// class TLoanReturnTracker { @@ -1620,10 +1620,10 @@ private: return DataToSend.empty(); } - bool Acked(ui64 dstTabletId) const { - return !DataToSend.contains(dstTabletId); - } - + bool Acked(ui64 dstTabletId) const { + return !DataToSend.contains(dstTabletId); + } + bool Has(ui64 dstTabletId, TActorId pipeClientActorId) const { return PipesToDstShards.contains(dstTabletId) && *PipesToDstShards.FindPtr(dstTabletId) == pipeClientActorId; } @@ -1641,141 +1641,141 @@ private: THashMap<ui64, TActorId> PipesToDstShards; }; - /// - class TChangeSenderActivator { - public: - explicit TChangeSenderActivator(ui64 selfTabletId) - : Origin(selfTabletId) + /// + class TChangeSenderActivator { + public: + explicit TChangeSenderActivator(ui64 selfTabletId) + : Origin(selfTabletId) , PipeRetryPolicy{ .RetryLimitCount = 20, .MinRetryTime = TDuration::MilliSeconds(10), .MaxRetryTime = TDuration::MilliSeconds(500), .BackoffMultiplier = 2 } - { - } - - void AddDst(ui64 dstTabletId) { - Dst.insert(dstTabletId); - } - - const THashSet<ui64>& GetDstSet() const { - return Dst; - } - - void DoSend(ui64 dstTabletId, const TActorContext& ctx) { - Y_VERIFY(Dst.contains(dstTabletId)); - NTabletPipe::TClientConfig clientConfig; - clientConfig.CheckAliveness = true; - clientConfig.RetryPolicy = PipeRetryPolicy; - PipesToDstShards[dstTabletId] = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, dstTabletId, clientConfig)); - - auto ev = MakeHolder<TEvChangeExchange::TEvActivateSender>(); - ev->Record.SetOrigin(Origin); - - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Activate change sender" - << ": origin# " << ev->Record.GetOrigin() - << ", dst# " << dstTabletId); - - NTabletPipe::SendData(ctx, PipesToDstShards[dstTabletId], ev.Release()); - } - - bool Ack(ui64 dstTabletId, const TActorContext &ctx) { - if (!Dst.contains(dstTabletId)) { - return false; - } - - NTabletPipe::CloseClient(ctx, PipesToDstShards[dstTabletId]); - PipesToDstShards.erase(dstTabletId); - Dst.erase(dstTabletId); - - return true; - } - - void AutoAck(ui64 dstTabletId, const TActorContext &ctx) { - if (!Ack(dstTabletId, ctx)) { - return; - } - - auto ev = MakeHolder<TEvChangeExchange::TEvActivateSenderAck>(); - ev->Record.SetOrigin(dstTabletId); - ctx.Send(ctx.SelfID, ev.Release()); - } - - bool AllAcked() const { - return Dst.empty(); - } - - bool Acked(ui64 dstTabletId) const { - return !Dst.contains(dstTabletId); - } - - bool Has(ui64 dstTabletId, TActorId pipeClientActorId) const { - auto it = PipesToDstShards.find(dstTabletId); - if (it == PipesToDstShards.end()) { - return false; - } - - return it->second == pipeClientActorId; - } - - void Shutdown(const TActorContext &ctx) { - for (const auto& p : PipesToDstShards) { - NTabletPipe::CloseClient(ctx, p.second); - } - } - - TString Dump() const { - return JoinSeq(", ", Dst); - } - - private: - const ui64 Origin; + { + } + + void AddDst(ui64 dstTabletId) { + Dst.insert(dstTabletId); + } + + const THashSet<ui64>& GetDstSet() const { + return Dst; + } + + void DoSend(ui64 dstTabletId, const TActorContext& ctx) { + Y_VERIFY(Dst.contains(dstTabletId)); + NTabletPipe::TClientConfig clientConfig; + clientConfig.CheckAliveness = true; + clientConfig.RetryPolicy = PipeRetryPolicy; + PipesToDstShards[dstTabletId] = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, dstTabletId, clientConfig)); + + auto ev = MakeHolder<TEvChangeExchange::TEvActivateSender>(); + ev->Record.SetOrigin(Origin); + + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Activate change sender" + << ": origin# " << ev->Record.GetOrigin() + << ", dst# " << dstTabletId); + + NTabletPipe::SendData(ctx, PipesToDstShards[dstTabletId], ev.Release()); + } + + bool Ack(ui64 dstTabletId, const TActorContext &ctx) { + if (!Dst.contains(dstTabletId)) { + return false; + } + + NTabletPipe::CloseClient(ctx, PipesToDstShards[dstTabletId]); + PipesToDstShards.erase(dstTabletId); + Dst.erase(dstTabletId); + + return true; + } + + void AutoAck(ui64 dstTabletId, const TActorContext &ctx) { + if (!Ack(dstTabletId, ctx)) { + return; + } + + auto ev = MakeHolder<TEvChangeExchange::TEvActivateSenderAck>(); + ev->Record.SetOrigin(dstTabletId); + ctx.Send(ctx.SelfID, ev.Release()); + } + + bool AllAcked() const { + return Dst.empty(); + } + + bool Acked(ui64 dstTabletId) const { + return !Dst.contains(dstTabletId); + } + + bool Has(ui64 dstTabletId, TActorId pipeClientActorId) const { + auto it = PipesToDstShards.find(dstTabletId); + if (it == PipesToDstShards.end()) { + return false; + } + + return it->second == pipeClientActorId; + } + + void Shutdown(const TActorContext &ctx) { + for (const auto& p : PipesToDstShards) { + NTabletPipe::CloseClient(ctx, p.second); + } + } + + TString Dump() const { + return JoinSeq(", ", Dst); + } + + private: + const ui64 Origin; NTabletPipe::TClientRetryPolicy PipeRetryPolicy; - - THashSet<ui64> Dst; - THashMap<ui64, TActorId> PipesToDstShards; - }; - - class TChangeExchangeSplitter { - public: - explicit TChangeExchangeSplitter(const TDataShard* self) - : Self(self) - { - } - - void AddDst(ui64 dstTabletId) { - DstTabletIds.insert(dstTabletId); - } - - void DoSplit(const TActorContext& ctx) { - Y_VERIFY(DstTabletIds); - Worker = ctx.Register(CreateChangeExchangeSplit(Self, TVector<ui64>(DstTabletIds.begin(), DstTabletIds.end()))); - Acked = false; - } - - void Ack() { - Acked = true; - } - - bool Done() const { - return !DstTabletIds || Acked; - } - - void Shutdown(const TActorContext& ctx) { - if (Worker) { - ctx.Send(std::exchange(Worker, TActorId()), new TEvents::TEvPoisonPill()); - } - } - - private: - const TDataShard* Self; - - THashSet<ui64> DstTabletIds; - TActorId Worker; - bool Acked = false; - }; - + + THashSet<ui64> Dst; + THashMap<ui64, TActorId> PipesToDstShards; + }; + + class TChangeExchangeSplitter { + public: + explicit TChangeExchangeSplitter(const TDataShard* self) + : Self(self) + { + } + + void AddDst(ui64 dstTabletId) { + DstTabletIds.insert(dstTabletId); + } + + void DoSplit(const TActorContext& ctx) { + Y_VERIFY(DstTabletIds); + Worker = ctx.Register(CreateChangeExchangeSplit(Self, TVector<ui64>(DstTabletIds.begin(), DstTabletIds.end()))); + Acked = false; + } + + void Ack() { + Acked = true; + } + + bool Done() const { + return !DstTabletIds || Acked; + } + + void Shutdown(const TActorContext& ctx) { + if (Worker) { + ctx.Send(std::exchange(Worker, TActorId()), new TEvents::TEvPoisonPill()); + } + } + + private: + const TDataShard* Self; + + THashSet<ui64> DstTabletIds; + TActorId Worker; + bool Acked = false; + }; + // For follower only struct TFollowerState { ui64 LastSchemeUpdate = 0; @@ -1927,7 +1927,7 @@ private: std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> DstSplitDescription; std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SrcSplitDescription; THashSet<TActorId> SrcAckSplitTo; - THashMap<TActorId, THashSet<ui64>> SrcAckPartitioningChangedTo; + THashMap<TActorId, THashSet<ui64>> SrcAckPartitioningChangedTo; const ui32 SysTablesToTransferAtSplit[4] = { Schema::TxMain::TableId, Schema::TxDetails::TableId, @@ -1941,8 +1941,8 @@ private: ui32 State; ui32 LastLocalTid; ui64 LastSeqno; - ui64 NextChangeRecordOrder; - ui64 LastChangeRecordGroup; + ui64 NextChangeRecordOrder; + ui64 LastChangeRecordGroup; ui64 TxReadSizeLimit; ui64 StatisticsDisabled; bool Stopping = false; @@ -1960,15 +1960,15 @@ private: TSysLocks SysLocks; TSnapshotManager SnapshotManager; - TSchemaSnapshotManager SchemaSnapshotManager; + TSchemaSnapshotManager SchemaSnapshotManager; TReplicationSourceOffsetsServerLink ReplicationSourceOffsetsServer; TBuildIndexManager BuildIndexManager; - TS3UploadsManager S3Uploads; - TS3DownloadsManager S3Downloads; - + TS3UploadsManager S3Uploads; + TS3DownloadsManager S3Downloads; + TIntrusivePtr<TMediatorTimecastEntry> MediatorTimeCastEntry; TSet<ui64> MediatorTimeCastWaitingSteps; @@ -1999,71 +1999,71 @@ private: // Simple volatile counter ui64 NextTieBreakerIndex = 1; - struct TInFlightCondErase { - ui64 TxId; - ui64 ScanId; + struct TInFlightCondErase { + ui64 TxId; + ui64 ScanId; TActorId ActorId; - ui32 Condition; - - TInFlightCondErase() { - Clear(); - } - - void Clear() { - TxId = 0; - ScanId = 0; + ui32 Condition; + + TInFlightCondErase() { + Clear(); + } + + void Clear() { + TxId = 0; + ScanId = 0; ActorId = TActorId(); - Condition = 0; - } - - void Enqueue(ui64 txId, ui64 scanId, ui32 condition) { - TxId = txId; - ScanId = scanId; - Condition = condition; - } - - explicit operator bool() const { - return bool(ScanId); - } - - bool IsActive() const { - return bool(ActorId); - } - }; - - TInFlightCondErase InFlightCondErase; - - /// change sending & receiving - struct TInChangeSender { - ui64 Generation; - ui64 LastRecordOrder; - - explicit TInChangeSender(ui64 generation, ui64 lastRecordOrder = 0) - : Generation(generation) - , LastRecordOrder(lastRecordOrder) - { - } - }; - - using TRequestedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; - - // split/merge - TChangeSenderActivator ChangeSenderActivator; - TChangeExchangeSplitter ChangeExchangeSplitter; - THashSet<ui64> ReceiveActivationsFrom; - - // out - THashMap<TActorId, TSet<TRequestedRecord>> ChangeRecordsRequested; - TSet<ui64> ChangeRecordsToRemove; // ui64 is order - bool RequestChangeRecordsInFly = false; - bool RemoveChangeRecordsInFly = false; - THashMap<ui64, ui64> ChangesQueue; // order to size - ui64 ChangesQueueBytes = 0; - TActorId OutChangeSender; - - // in - THashMap<ui64, TInChangeSender> InChangeSenders; // ui64 is shard id - + Condition = 0; + } + + void Enqueue(ui64 txId, ui64 scanId, ui32 condition) { + TxId = txId; + ScanId = scanId; + Condition = condition; + } + + explicit operator bool() const { + return bool(ScanId); + } + + bool IsActive() const { + return bool(ActorId); + } + }; + + TInFlightCondErase InFlightCondErase; + + /// change sending & receiving + struct TInChangeSender { + ui64 Generation; + ui64 LastRecordOrder; + + explicit TInChangeSender(ui64 generation, ui64 lastRecordOrder = 0) + : Generation(generation) + , LastRecordOrder(lastRecordOrder) + { + } + }; + + using TRequestedRecord = TEvChangeExchange::TEvRequestRecords::TRecordInfo; + + // split/merge + TChangeSenderActivator ChangeSenderActivator; + TChangeExchangeSplitter ChangeExchangeSplitter; + THashSet<ui64> ReceiveActivationsFrom; + + // out + THashMap<TActorId, TSet<TRequestedRecord>> ChangeRecordsRequested; + TSet<ui64> ChangeRecordsToRemove; // ui64 is order + bool RequestChangeRecordsInFly = false; + bool RemoveChangeRecordsInFly = false; + THashMap<ui64, ui64> ChangesQueue; // order to size + ui64 ChangesQueueBytes = 0; + TActorId OutChangeSender; + + // in + THashMap<ui64, TInChangeSender> InChangeSenders; // ui64 is shard id + // compactionId, tableId/ownerId, actorId using TCompactionWaiter = std::tuple<ui64, TPathId, TActorId>; using TCompactionWaiterList = TList<TCompactionWaiter>; @@ -2138,13 +2138,13 @@ protected: HFuncTraced(TEvDataShard::TEvProposeTransaction, Handle); HFuncTraced(TEvDataShard::TEvProposeTransactionAttach, Handle); HFuncTraced(TEvDataShard::TEvCancelBackup, Handle); - HFuncTraced(TEvDataShard::TEvCancelRestore, Handle); - HFuncTraced(TEvDataShard::TEvGetS3Upload, Handle); - HFuncTraced(TEvDataShard::TEvStoreS3UploadId, Handle); - HFuncTraced(TEvDataShard::TEvChangeS3UploadStatus, Handle); - HFuncTraced(TEvDataShard::TEvGetS3DownloadInfo, Handle); - HFuncTraced(TEvDataShard::TEvStoreS3DownloadInfo, Handle); - HFuncTraced(TEvDataShard::TEvUnsafeUploadRowsRequest, Handle); + HFuncTraced(TEvDataShard::TEvCancelRestore, Handle); + HFuncTraced(TEvDataShard::TEvGetS3Upload, Handle); + HFuncTraced(TEvDataShard::TEvStoreS3UploadId, Handle); + HFuncTraced(TEvDataShard::TEvChangeS3UploadStatus, Handle); + HFuncTraced(TEvDataShard::TEvGetS3DownloadInfo, Handle); + HFuncTraced(TEvDataShard::TEvStoreS3DownloadInfo, Handle); + HFuncTraced(TEvDataShard::TEvUnsafeUploadRowsRequest, Handle); HFuncTraced(TEvDataShard::TEvMigrateSchemeShardRequest, Handle); HFuncTraced(TEvTxProcessing::TEvPlanStep, Handle); HFuncTraced(TEvTxProcessing::TEvReadSet, Handle); @@ -2181,9 +2181,9 @@ protected: HFunc(TEvDataShard::TEvS3ListingRequest, Handle); HFunc(TEvDataShard::TEvKqpScan, Handle); HFunc(TEvDataShard::TEvUploadRowsRequest, Handle); - HFunc(TEvDataShard::TEvEraseRowsRequest, Handle); - HFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle); - HFunc(TEvPrivate::TEvConditionalEraseRowsRegistered, Handle); + HFunc(TEvDataShard::TEvEraseRowsRequest, Handle); + HFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle); + HFunc(TEvPrivate::TEvConditionalEraseRowsRegistered, Handle); HFunc(TEvDataShard::TEvRead, Handle); HFunc(TEvDataShard::TEvReadContinue, Handle); HFunc(TEvDataShard::TEvReadAck, Handle); @@ -2201,7 +2201,7 @@ protected: HFunc(TEvDataShard::TEvRefreshVolatileSnapshotRequest, Handle); HFunc(TEvDataShard::TEvDiscardVolatileSnapshotRequest, Handle); HFuncTraced(TEvDataShard::TEvBuildIndexCreateRequest, Handle); - HFunc(TEvPrivate::TEvAsyncJobComplete, Handle); + HFunc(TEvPrivate::TEvAsyncJobComplete, Handle); CFunc(TEvPrivate::EvPeriodicWakeup, DoPeriodicTasks); HFunc(TEvents::TEvUndelivered, Handle); IgnoreFunc(TEvInterconnect::TEvNodeConnected); @@ -2210,15 +2210,15 @@ protected: HFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted); IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable); - HFunc(TEvChangeExchange::TEvRequestRecords, Handle); - HFunc(TEvChangeExchange::TEvRemoveRecords, Handle); - HFunc(TEvPrivate::TEvRequestChangeRecords, Handle); - HFunc(TEvPrivate::TEvRemoveChangeRecords, Handle); - HFunc(TEvChangeExchange::TEvHandshake, Handle); - HFunc(TEvChangeExchange::TEvApplyRecords, Handle); - HFunc(TEvChangeExchange::TEvActivateSender, Handle); - HFunc(TEvChangeExchange::TEvActivateSenderAck, Handle); - HFunc(TEvChangeExchange::TEvSplitAck, Handle); + HFunc(TEvChangeExchange::TEvRequestRecords, Handle); + HFunc(TEvChangeExchange::TEvRemoveRecords, Handle); + HFunc(TEvPrivate::TEvRequestChangeRecords, Handle); + HFunc(TEvPrivate::TEvRemoveChangeRecords, Handle); + HFunc(TEvChangeExchange::TEvHandshake, Handle); + HFunc(TEvChangeExchange::TEvApplyRecords, Handle); + HFunc(TEvChangeExchange::TEvActivateSender, Handle); + HFunc(TEvChangeExchange::TEvActivateSenderAck, Handle); + HFunc(TEvChangeExchange::TEvSplitAck, Handle); HFunc(TEvDataShard::TEvGetRemovedRowVersions, Handle); HFunc(TEvDataShard::TEvCompactBorrowed, Handle); HFunc(TEvDataShard::TEvCompactTable, Handle); @@ -2287,10 +2287,10 @@ protected: } Actors.clear(); - KillChangeSender(ctx); - ChangeSenderActivator.Shutdown(ctx); - ChangeExchangeSplitter.Shutdown(ctx); - + KillChangeSender(ctx); + ChangeSenderActivator.Shutdown(ctx); + ChangeExchangeSplitter.Shutdown(ctx); + StopFindSubDomainPathId(); StopWatchingSubDomainPathId(); diff --git a/ydb/core/tx/datashard/datashard_loans.cpp b/ydb/core/tx/datashard/datashard_loans.cpp index c45465bf740..adbfc4769db 100644 --- a/ydb/core/tx/datashard/datashard_loans.cpp +++ b/ydb/core/tx/datashard/datashard_loans.cpp @@ -2,8 +2,8 @@ #include <ydb/core/tablet_flat/tablet_flat_executor.h> -#include <util/string/join.h> - +#include <util/string/join.h> + namespace NKikimr { namespace NDataShard { @@ -208,22 +208,22 @@ void TDataShard::CheckStateChange(const TActorContext& ctx) { return str.Str(); }; - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, TabletID() << " in PreOffline state" - << " HasSharedBobs: " << HasSharedBlobs() + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, TabletID() << " in PreOffline state" + << " HasSharedBobs: " << HasSharedBlobs() << " SchemaOperations: " << fnListTxIds(TransQueue.GetSchemaOperations()) - << " OutReadSets count: " << OutReadSets.CountReadSets() - << " ChangesQueue size: " << ChangesQueue.size() - << " ChangeExchangeSplit: " << ChangeExchangeSplitter.Done() - << " siblings to be activated: " << ChangeSenderActivator.Dump() - << " wait to activation from: " << JoinSeq(", ", ReceiveActivationsFrom)); - - const bool hasSharedBlobs = HasSharedBlobs(); - const bool hasSchemaOps = !TransQueue.GetSchemaOperations().empty(); - const bool hasOutRs = !OutReadSets.Empty(); - const bool hasChangeRecords = !ChangesQueue.empty(); - const bool mustActivateOthers = !ChangeSenderActivator.AllAcked(); - - if (!hasSharedBlobs && !hasSchemaOps && !hasOutRs && !hasChangeRecords && !mustActivateOthers) { + << " OutReadSets count: " << OutReadSets.CountReadSets() + << " ChangesQueue size: " << ChangesQueue.size() + << " ChangeExchangeSplit: " << ChangeExchangeSplitter.Done() + << " siblings to be activated: " << ChangeSenderActivator.Dump() + << " wait to activation from: " << JoinSeq(", ", ReceiveActivationsFrom)); + + const bool hasSharedBlobs = HasSharedBlobs(); + const bool hasSchemaOps = !TransQueue.GetSchemaOperations().empty(); + const bool hasOutRs = !OutReadSets.Empty(); + const bool hasChangeRecords = !ChangesQueue.empty(); + const bool mustActivateOthers = !ChangeSenderActivator.AllAcked(); + + if (!hasSharedBlobs && !hasSchemaOps && !hasOutRs && !hasChangeRecords && !mustActivateOthers) { Y_VERIFY(!TxInFly()); Execute(new TTxGoOffline(this), ctx); } diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp index 65412f7319d..3498162c17e 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.cpp +++ b/ydb/core/tx/datashard/datashard_pipeline.cpp @@ -111,9 +111,9 @@ bool TPipeline::CanRunOp(const TOperation &op) const { if (!op.Exists()) return false; - const TSchemaOperation *schemaOp = Self->TransQueue.FindSchemaTx(op.GetTxId()); + const TSchemaOperation *schemaOp = Self->TransQueue.FindSchemaTx(op.GetTxId()); if (schemaOp) { - if (schemaOp->Type == TSchemaOperation::ETypeDrop && !Self->CanDrop()) { + if (schemaOp->Type == TSchemaOperation::ETypeDrop && !Self->CanDrop()) { return false; } } @@ -768,13 +768,13 @@ void TPipeline::RemoveTx(TStepOrder stepTxId) { ForgetTx(stepTxId.TxId); } -const TSchemaOperation* TPipeline::FindSchemaTx(ui64 txId) const { +const TSchemaOperation* TPipeline::FindSchemaTx(ui64 txId) const { return Self->TransQueue.FindSchemaTx(txId); } void TPipeline::CompleteSchemaTx(NIceDb::TNiceDb& db, ui64 txId) { Y_VERIFY(txId); - TSchemaOperation * op = Self->TransQueue.FindSchemaTx(txId); + TSchemaOperation * op = Self->TransQueue.FindSchemaTx(txId); if (!op) return; @@ -1005,7 +1005,7 @@ void TPipeline::UpdateSchemeTxBody(ui64 txId, const TStringBuf &txBody, TTransac Self->TransQueue.UpdateTxBody(db, txId, txBody); } -void TPipeline::ProposeSchemeTx(const TSchemaOperation &op, +void TPipeline::ProposeSchemeTx(const TSchemaOperation &op, TTransactionContext &txc) { NIceDb::TNiceDb db(txc.DB); @@ -1141,21 +1141,21 @@ TOperation::TPtr TPipeline::BuildOperation(TEvDataShard::TEvProposeTransaction:: tx->SetTxBody(rec.GetTxBody()); tx->SetCookie(ev->Cookie); - auto malformed = [&](const TStringBuf txType, const TString& txBody) { - const TString error = TStringBuilder() << "Malformed " << txType << " tx" - << " at tablet " << Self->TabletID() - << " txId " << tx->GetTxId() - << " ssId " << tx->GetSchemeShardId() - << " parsed tx body " << txBody; - - tx->SetAbortedFlag(); + auto malformed = [&](const TStringBuf txType, const TString& txBody) { + const TString error = TStringBuilder() << "Malformed " << txType << " tx" + << " at tablet " << Self->TabletID() + << " txId " << tx->GetTxId() + << " ssId " << tx->GetSchemeShardId() + << " parsed tx body " << txBody; + + tx->SetAbortedFlag(); tx->Result().Reset(new TEvDataShard::TEvProposeTransactionResult( rec.GetTxKind(), Self->TabletID(), tx->GetTxId(), NKikimrTxDataShard::TEvProposeTransactionResult::ERROR)); - tx->Result()->SetProcessError(NKikimrTxDataShard::TError::BAD_ARGUMENT, error); - - LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, error); - }; - + tx->Result()->SetProcessError(NKikimrTxDataShard::TError::BAD_ARGUMENT, error); + + LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, error); + }; + if (tx->IsSchemeTx()) { Y_VERIFY(rec.HasSchemeShardId()); Y_VERIFY(rec.HasProcessingParams()); @@ -1193,16 +1193,16 @@ TOperation::TPtr TPipeline::BuildOperation(TEvDataShard::TEvProposeTransaction:: tx->SetReadOnlyFlag(); tx->SetGlobalReaderFlag(); } - } else if (tx->IsDistributedEraseTx()) { - if (!tx->BuildDistributedEraseTx()) { + } else if (tx->IsDistributedEraseTx()) { + if (!tx->BuildDistributedEraseTx()) { malformed(TStringBuf("distributed erase"), tx->GetDistributedEraseTx()->GetBody().ShortDebugString()); - return tx; - } - - tx->SetGlobalWriterFlag(); - if (tx->GetDistributedEraseTx()->HasDependents()) { - tx->SetGlobalReaderFlag(); - } + return tx; + } + + tx->SetGlobalWriterFlag(); + if (tx->GetDistributedEraseTx()->HasDependents()) { + tx->SetGlobalReaderFlag(); + } } else if (tx->IsCommitWritesTx()) { if (!tx->BuildCommitWritesTx()) { malformed(TStringBuf("commit writes"), tx->GetCommitWritesTx()->GetBody().ShortDebugString()); diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h index 8d03ea72bfa..67c35260b28 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.h +++ b/ydb/core/tx/datashard/datashard_pipeline.h @@ -122,7 +122,7 @@ public: void DeactivateOp(TOperation::TPtr op, TTransactionContext& txc, const TActorContext &ctx); void RemoveTx(TStepOrder stepTxId); - const TSchemaOperation* FindSchemaTx(ui64 txId) const; + const TSchemaOperation* FindSchemaTx(ui64 txId) const; void CompleteSchemaTx(NIceDb::TNiceDb& db, ui64 txId); void MarkOpAsUsingSnapshot(TOperation::TPtr op); @@ -155,7 +155,7 @@ public: bool HasAlter() const { return SchemaTx && SchemaTx->IsAlter(); } bool HasDrop() const { return SchemaTx && SchemaTx->IsDrop(); } bool HasBackup() const { return SchemaTx && SchemaTx->IsBackup(); } - bool HasRestore() const { return SchemaTx && SchemaTx->IsRestore(); } + bool HasRestore() const { return SchemaTx && SchemaTx->IsRestore(); } bool HasCopy() const { return SchemaTx && SchemaTx->IsCopy(); } bool HasCreatePersistentSnapshot() const { return SchemaTx && SchemaTx->IsCreatePersistentSnapshot(); } bool HasDropPersistentSnapshot() const { return SchemaTx && SchemaTx->IsDropPersistentSnapshot(); } @@ -163,9 +163,9 @@ public: bool HasFinalizeBuilIndex() const { return SchemaTx && SchemaTx->IsFinalizeBuildIndex(); } bool HasDropIndexNotice() const { return SchemaTx && SchemaTx->IsDropIndexNotice(); } bool HasMove() const { return SchemaTx && SchemaTx->IsMove(); } - bool HasCreateCdcStream() const { return SchemaTx && SchemaTx->IsCreateCdcStream(); } - bool HasAlterCdcStream() const { return SchemaTx && SchemaTx->IsAlterCdcStream(); } - bool HasDropCdcStream() const { return SchemaTx && SchemaTx->IsDropCdcStream(); } + bool HasCreateCdcStream() const { return SchemaTx && SchemaTx->IsCreateCdcStream(); } + bool HasAlterCdcStream() const { return SchemaTx && SchemaTx->IsAlterCdcStream(); } + bool HasDropCdcStream() const { return SchemaTx && SchemaTx->IsDropCdcStream(); } ui64 CurrentSchemaTxId() const { if (SchemaTx) @@ -173,11 +173,11 @@ public: return 0; } - const TSchemaOperation* GetSchemaOp() const { + const TSchemaOperation* GetSchemaOp() const { return SchemaTx; } - void SetSchemaOp(TSchemaOperation * op) { + void SetSchemaOp(TSchemaOperation * op) { Y_VERIFY(!SchemaTx || SchemaTx->TxId == op->TxId); SchemaTx = op; } @@ -188,7 +188,7 @@ public: void ProposeComplete(const TOperation::TPtr &op, const TActorContext &ctx); void PersistTxFlags(TOperation::TPtr op, TTransactionContext &txc); void UpdateSchemeTxBody(ui64 txId, const TStringBuf &txBody, TTransactionContext &txc); - void ProposeSchemeTx(const TSchemaOperation &op, TTransactionContext &txc); + void ProposeSchemeTx(const TSchemaOperation &op, TTransactionContext &txc); bool CancelPropose(NIceDb::TNiceDb& db, const TActorContext& ctx, ui64 txId); ECleanupStatus CleanupOutdated(NIceDb::TNiceDb& db, const TActorContext& ctx, ui64 outdatedStep); ui64 PlannedTxInFly() const; @@ -437,7 +437,7 @@ private: TStepOrder UtmostCompleteTx; ui64 KeepSchemaStep; ui64 LastCleanupTime; - TSchemaOperation * SchemaTx; + TSchemaOperation * SchemaTx; std::array<THolder<TExecutionUnit>, (ui32)EExecutionUnitKind::Count> ExecutionUnits; THashSet<TOperation::TPtr> ExecuteBlockers; // Candidates for execution. diff --git a/ydb/core/tx/datashard/datashard_s3_downloads.cpp b/ydb/core/tx/datashard/datashard_s3_downloads.cpp index c1b65505e63..f1571885c50 100644 --- a/ydb/core/tx/datashard/datashard_s3_downloads.cpp +++ b/ydb/core/tx/datashard/datashard_s3_downloads.cpp @@ -1,67 +1,67 @@ -#include "datashard_s3_downloads.h" -#include "datashard_impl.h" - -namespace NKikimr { +#include "datashard_s3_downloads.h" +#include "datashard_impl.h" + +namespace NKikimr { namespace NDataShard { - -bool TS3DownloadsManager::Load(NIceDb::TNiceDb& db) { + +bool TS3DownloadsManager::Load(NIceDb::TNiceDb& db) { using Schema = TDataShard::Schema; - - bool ready = true; - auto rowset = db.Table<Schema::S3Downloads>().Range().Select(); - if (rowset.IsReady()) { - while (!rowset.EndOfSet()) { - ui64 txId = rowset.GetValue<Schema::S3Downloads::TxId>(); - - Y_VERIFY_S(!Downloads.contains(txId), "Unexpected duplicate s3 download: " << txId); - auto& info = Downloads[txId]; - - if (rowset.HaveValue<Schema::S3Downloads::DataETag>()) { - info.DataETag = rowset.GetValue<Schema::S3Downloads::DataETag>(); - } - - info.ProcessedBytes = rowset.GetValueOrDefault<Schema::S3Downloads::ProcessedBytes>(0); - info.WrittenBytes = rowset.GetValueOrDefault<Schema::S3Downloads::WrittenBytes>(0); - info.WrittenRows = rowset.GetValueOrDefault<Schema::S3Downloads::WrittenRows>(0); - - if (!rowset.Next()) { - ready = false; - break; - } - } - } else { - ready = false; - } - - return ready; -} - -void TS3DownloadsManager::Reset() { - Downloads.clear(); -} - -const TS3DownloadsManager::TInfo* TS3DownloadsManager::Find(ui64 txId) const { - return Downloads.FindPtr(txId); -} - -const TS3DownloadsManager::TInfo& TS3DownloadsManager::Store(NIceDb::TNiceDb& db, const TEvDataShard::TEvStoreS3DownloadInfo& msg) { - auto& info = Downloads[msg.TxId]; - - Y_VERIFY(info.DataETag.GetOrElse(msg.DataETag) == msg.DataETag); - info.DataETag = msg.DataETag; - info.ProcessedBytes = msg.ProcessedBytes; - info.WrittenBytes = msg.WrittenBytes; - info.WrittenRows = msg.WrittenRows; - + + bool ready = true; + auto rowset = db.Table<Schema::S3Downloads>().Range().Select(); + if (rowset.IsReady()) { + while (!rowset.EndOfSet()) { + ui64 txId = rowset.GetValue<Schema::S3Downloads::TxId>(); + + Y_VERIFY_S(!Downloads.contains(txId), "Unexpected duplicate s3 download: " << txId); + auto& info = Downloads[txId]; + + if (rowset.HaveValue<Schema::S3Downloads::DataETag>()) { + info.DataETag = rowset.GetValue<Schema::S3Downloads::DataETag>(); + } + + info.ProcessedBytes = rowset.GetValueOrDefault<Schema::S3Downloads::ProcessedBytes>(0); + info.WrittenBytes = rowset.GetValueOrDefault<Schema::S3Downloads::WrittenBytes>(0); + info.WrittenRows = rowset.GetValueOrDefault<Schema::S3Downloads::WrittenRows>(0); + + if (!rowset.Next()) { + ready = false; + break; + } + } + } else { + ready = false; + } + + return ready; +} + +void TS3DownloadsManager::Reset() { + Downloads.clear(); +} + +const TS3DownloadsManager::TInfo* TS3DownloadsManager::Find(ui64 txId) const { + return Downloads.FindPtr(txId); +} + +const TS3DownloadsManager::TInfo& TS3DownloadsManager::Store(NIceDb::TNiceDb& db, const TEvDataShard::TEvStoreS3DownloadInfo& msg) { + auto& info = Downloads[msg.TxId]; + + Y_VERIFY(info.DataETag.GetOrElse(msg.DataETag) == msg.DataETag); + info.DataETag = msg.DataETag; + info.ProcessedBytes = msg.ProcessedBytes; + info.WrittenBytes = msg.WrittenBytes; + info.WrittenRows = msg.WrittenRows; + using Schema = TDataShard::Schema; - db.Table<Schema::S3Downloads>().Key(msg.TxId).Update( - NIceDb::TUpdate<Schema::S3Downloads::DataETag>(msg.DataETag), - NIceDb::TUpdate<Schema::S3Downloads::ProcessedBytes>(msg.ProcessedBytes), - NIceDb::TUpdate<Schema::S3Downloads::WrittenBytes>(msg.WrittenBytes), - NIceDb::TUpdate<Schema::S3Downloads::WrittenRows>(msg.WrittenRows)); - - return info; -} - + db.Table<Schema::S3Downloads>().Key(msg.TxId).Update( + NIceDb::TUpdate<Schema::S3Downloads::DataETag>(msg.DataETag), + NIceDb::TUpdate<Schema::S3Downloads::ProcessedBytes>(msg.ProcessedBytes), + NIceDb::TUpdate<Schema::S3Downloads::WrittenBytes>(msg.WrittenBytes), + NIceDb::TUpdate<Schema::S3Downloads::WrittenRows>(msg.WrittenRows)); + + return info; +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_s3_downloads.h b/ydb/core/tx/datashard/datashard_s3_downloads.h index f4d0fbfc6e6..d7d11ad2f21 100644 --- a/ydb/core/tx/datashard/datashard_s3_downloads.h +++ b/ydb/core/tx/datashard/datashard_s3_downloads.h @@ -1,28 +1,28 @@ -#pragma once - -#include "datashard.h" - +#pragma once + +#include "datashard.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -#include <util/generic/hash.h> - -namespace NKikimr { + +#include <util/generic/hash.h> + +namespace NKikimr { namespace NDataShard { - -class TS3DownloadsManager { - using TInfo = TEvDataShard::TEvS3DownloadInfo::TInfo; - -public: - bool Load(NIceDb::TNiceDb& db); - void Reset(); - - const TInfo* Find(ui64 txId) const; - const TInfo& Store(NIceDb::TNiceDb& db, const TEvDataShard::TEvStoreS3DownloadInfo& msg); - -private: - THashMap<ui64, TInfo> Downloads; - -}; // TS3DownloadsManager - + +class TS3DownloadsManager { + using TInfo = TEvDataShard::TEvS3DownloadInfo::TInfo; + +public: + bool Load(NIceDb::TNiceDb& db); + void Reset(); + + const TInfo* Find(ui64 txId) const; + const TInfo& Store(NIceDb::TNiceDb& db, const TEvDataShard::TEvStoreS3DownloadInfo& msg); + +private: + THashMap<ui64, TInfo> Downloads; + +}; // TS3DownloadsManager + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_s3_upload.h b/ydb/core/tx/datashard/datashard_s3_upload.h index ad38cb0b3ae..ba991de863a 100644 --- a/ydb/core/tx/datashard/datashard_s3_upload.h +++ b/ydb/core/tx/datashard/datashard_s3_upload.h @@ -1,45 +1,45 @@ -#pragma once - -#include <util/generic/maybe.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/string/join.h> - -namespace NKikimr { +#pragma once + +#include <util/generic/maybe.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/string/join.h> + +namespace NKikimr { namespace NDataShard { - -struct TS3Upload { - enum class EStatus: ui8 { - UploadParts, - Complete, - Abort, - }; - - explicit TS3Upload(const TString& id) - : Id(id) - , Status(EStatus::UploadParts) - { - } - - TString Id; - EStatus Status; - TMaybe<TString> Error; - TVector<TString> Parts; - - void Out(IOutputStream& out) const { - out << "{" - << " Id: " << Id - << " Status: " << Status - << " Error: " << Error - << " Parts: [" << JoinSeq(",", Parts) << "]" - << " }"; - } - -}; // TS3Upload - + +struct TS3Upload { + enum class EStatus: ui8 { + UploadParts, + Complete, + Abort, + }; + + explicit TS3Upload(const TString& id) + : Id(id) + , Status(EStatus::UploadParts) + { + } + + TString Id; + EStatus Status; + TMaybe<TString> Error; + TVector<TString> Parts; + + void Out(IOutputStream& out) const { + out << "{" + << " Id: " << Id + << " Status: " << Status + << " Error: " << Error + << " Parts: [" << JoinSeq(",", Parts) << "]" + << " }"; + } + +}; // TS3Upload + } // namespace NDataShard -} // namespace NKikimr - +} // namespace NKikimr + Y_DECLARE_OUT_SPEC(inline, NKikimr::NDataShard::TS3Upload, out, value) { - value.Out(out); -} + value.Out(out); +} diff --git a/ydb/core/tx/datashard/datashard_s3_uploads.cpp b/ydb/core/tx/datashard/datashard_s3_uploads.cpp index 743b41df475..3915bf9d2e5 100644 --- a/ydb/core/tx/datashard/datashard_s3_uploads.cpp +++ b/ydb/core/tx/datashard/datashard_s3_uploads.cpp @@ -1,125 +1,125 @@ -#include "datashard_s3_uploads.h" -#include "datashard_impl.h" - -namespace NKikimr { +#include "datashard_s3_uploads.h" +#include "datashard_impl.h" + +namespace NKikimr { namespace NDataShard { - -bool TS3UploadsManager::Load(NIceDb::TNiceDb& db) { + +bool TS3UploadsManager::Load(NIceDb::TNiceDb& db) { using Schema = TDataShard::Schema; - - { - auto rowset = db.Table<Schema::S3Uploads>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const ui64 txId = rowset.GetValue<Schema::S3Uploads::TxId>(); - const TString uploadId = rowset.GetValue<Schema::S3Uploads::UploadId>(); - - auto res = Uploads.emplace(txId, TS3Upload(uploadId)); - Y_VERIFY_S(res.second, "Unexpected duplicate s3 upload" - << ": txId# " << txId - << ", uploadId# " << uploadId); - - auto& upload = res.first->second; - - upload.Status = rowset.GetValue<Schema::S3Uploads::Status>(); - if (rowset.HaveValue<Schema::S3Uploads::Error>()) { - upload.Error = rowset.GetValue<Schema::S3Uploads::Error>(); - } - - if (!rowset.Next()) { - return false; - } - } - } - - if (db.HaveTable<Schema::S3UploadedParts>()) { - auto rowset = db.Table<Schema::S3UploadedParts>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const ui64 txId = rowset.GetValue<Schema::S3UploadedParts::TxId>(); - - auto it = Uploads.find(txId); - Y_VERIFY_S(it != Uploads.end(), "Unknown s3 upload part" - << ": txId# " << txId); - - auto& parts = it->second.Parts; - - const ui32 partNumber = rowset.GetValue<Schema::S3UploadedParts::PartNumber>(); - if (parts.size() < partNumber) { - parts.resize(partNumber); - } - - parts[partNumber - 1] = rowset.GetValue<Schema::S3UploadedParts::ETag>(); - - if (!rowset.Next()) { - return false; - } - } - } - - return true; -} - -void TS3UploadsManager::Reset() { - Uploads.clear(); -} - -const TS3Upload* TS3UploadsManager::Find(ui64 txId) const { - auto it = Uploads.find(txId); - if (it == Uploads.end()) { - return nullptr; - } - - return &it->second; -} - -const TS3Upload& TS3UploadsManager::Add(NIceDb::TNiceDb& db, ui64 txId, const TString& uploadId) { + + { + auto rowset = db.Table<Schema::S3Uploads>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const ui64 txId = rowset.GetValue<Schema::S3Uploads::TxId>(); + const TString uploadId = rowset.GetValue<Schema::S3Uploads::UploadId>(); + + auto res = Uploads.emplace(txId, TS3Upload(uploadId)); + Y_VERIFY_S(res.second, "Unexpected duplicate s3 upload" + << ": txId# " << txId + << ", uploadId# " << uploadId); + + auto& upload = res.first->second; + + upload.Status = rowset.GetValue<Schema::S3Uploads::Status>(); + if (rowset.HaveValue<Schema::S3Uploads::Error>()) { + upload.Error = rowset.GetValue<Schema::S3Uploads::Error>(); + } + + if (!rowset.Next()) { + return false; + } + } + } + + if (db.HaveTable<Schema::S3UploadedParts>()) { + auto rowset = db.Table<Schema::S3UploadedParts>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const ui64 txId = rowset.GetValue<Schema::S3UploadedParts::TxId>(); + + auto it = Uploads.find(txId); + Y_VERIFY_S(it != Uploads.end(), "Unknown s3 upload part" + << ": txId# " << txId); + + auto& parts = it->second.Parts; + + const ui32 partNumber = rowset.GetValue<Schema::S3UploadedParts::PartNumber>(); + if (parts.size() < partNumber) { + parts.resize(partNumber); + } + + parts[partNumber - 1] = rowset.GetValue<Schema::S3UploadedParts::ETag>(); + + if (!rowset.Next()) { + return false; + } + } + } + + return true; +} + +void TS3UploadsManager::Reset() { + Uploads.clear(); +} + +const TS3Upload* TS3UploadsManager::Find(ui64 txId) const { + auto it = Uploads.find(txId); + if (it == Uploads.end()) { + return nullptr; + } + + return &it->second; +} + +const TS3Upload& TS3UploadsManager::Add(NIceDb::TNiceDb& db, ui64 txId, const TString& uploadId) { using Schema = TDataShard::Schema; - - Y_VERIFY(!Uploads.contains(txId)); - auto res = Uploads.emplace(txId, TS3Upload(uploadId)); - - db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::UploadId>(uploadId); - - return res.first->second; -} - -const TS3Upload& TS3UploadsManager::ChangeStatus(NIceDb::TNiceDb& db, ui64 txId, TS3Upload::EStatus status, - TMaybe<TString>&& error, TVector<TString>&& parts) -{ + + Y_VERIFY(!Uploads.contains(txId)); + auto res = Uploads.emplace(txId, TS3Upload(uploadId)); + + db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::UploadId>(uploadId); + + return res.first->second; +} + +const TS3Upload& TS3UploadsManager::ChangeStatus(NIceDb::TNiceDb& db, ui64 txId, TS3Upload::EStatus status, + TMaybe<TString>&& error, TVector<TString>&& parts) +{ using Schema = TDataShard::Schema; - - auto it = Uploads.find(txId); - Y_VERIFY(it != Uploads.end()); - - auto& upload = it->second; - upload.Status = status; - upload.Error = std::move(error); - upload.Parts = std::move(parts); - - db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::Status>(upload.Status); - - if (upload.Error) { - db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::Error>(*upload.Error); - } - - for (ui32 partIndex = 0; partIndex < upload.Parts.size(); ++partIndex) { - const ui32 partNumber = partIndex + 1; - const auto& eTag = upload.Parts.at(partIndex); - - db.Table<Schema::S3UploadedParts>().Key(txId, partNumber).Update( - NIceDb::TUpdate<Schema::S3UploadedParts::ETag>(eTag) - ); - } - - return upload; -} - + + auto it = Uploads.find(txId); + Y_VERIFY(it != Uploads.end()); + + auto& upload = it->second; + upload.Status = status; + upload.Error = std::move(error); + upload.Parts = std::move(parts); + + db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::Status>(upload.Status); + + if (upload.Error) { + db.Table<Schema::S3Uploads>().Key(txId).Update<Schema::S3Uploads::Error>(*upload.Error); + } + + for (ui32 partIndex = 0; partIndex < upload.Parts.size(); ++partIndex) { + const ui32 partNumber = partIndex + 1; + const auto& eTag = upload.Parts.at(partIndex); + + db.Table<Schema::S3UploadedParts>().Key(txId, partNumber).Update( + NIceDb::TUpdate<Schema::S3UploadedParts::ETag>(eTag) + ); + } + + return upload; +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_s3_uploads.h b/ydb/core/tx/datashard/datashard_s3_uploads.h index 741a01b4031..1755327cc5f 100644 --- a/ydb/core/tx/datashard/datashard_s3_uploads.h +++ b/ydb/core/tx/datashard/datashard_s3_uploads.h @@ -1,42 +1,42 @@ -#pragma once - -#include "datashard_s3_upload.h" - +#pragma once + +#include "datashard_s3_upload.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -#include <util/generic/hash.h> -#include <util/generic/string.h> - -namespace NKikimr { + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +namespace NKikimr { namespace NDataShard { - -class TS3UploadsManager { -public: - bool Load(NIceDb::TNiceDb& db); - void Reset(); - - const TS3Upload* Find(ui64 txId) const; - - const TS3Upload& Add(NIceDb::TNiceDb& db, ui64 txId, const TString& uploadId); - const TS3Upload& Add(NTable::TDatabase& db, ui64 txId, const TString& uploadId) { - NIceDb::TNiceDb nicedb(db); - return Add(nicedb, txId, uploadId); - } - - const TS3Upload& ChangeStatus(NIceDb::TNiceDb& db, ui64 txId, TS3Upload::EStatus status, - TMaybe<TString>&& error, TVector<TString>&& parts); - - const TS3Upload& ChangeStatus(NTable::TDatabase& db, ui64 txId, TS3Upload::EStatus status, - TMaybe<TString>&& error, TVector<TString>&& parts) - { - NIceDb::TNiceDb nicedb(db); - return ChangeStatus(nicedb, txId, status, std::move(error), std::move(parts)); - } - -private: - THashMap<ui64, TS3Upload> Uploads; - -}; // TS3UploadsManager - + +class TS3UploadsManager { +public: + bool Load(NIceDb::TNiceDb& db); + void Reset(); + + const TS3Upload* Find(ui64 txId) const; + + const TS3Upload& Add(NIceDb::TNiceDb& db, ui64 txId, const TString& uploadId); + const TS3Upload& Add(NTable::TDatabase& db, ui64 txId, const TString& uploadId) { + NIceDb::TNiceDb nicedb(db); + return Add(nicedb, txId, uploadId); + } + + const TS3Upload& ChangeStatus(NIceDb::TNiceDb& db, ui64 txId, TS3Upload::EStatus status, + TMaybe<TString>&& error, TVector<TString>&& parts); + + const TS3Upload& ChangeStatus(NTable::TDatabase& db, ui64 txId, TS3Upload::EStatus status, + TMaybe<TString>&& error, TVector<TString>&& parts) + { + NIceDb::TNiceDb nicedb(db); + return ChangeStatus(nicedb, txId, status, std::move(error), std::move(parts)); + } + +private: + THashMap<ui64, TS3Upload> Uploads; + +}; // TS3UploadsManager + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_schema_snapshots.cpp b/ydb/core/tx/datashard/datashard_schema_snapshots.cpp index 3e104dbafa0..d4c7ab05106 100644 --- a/ydb/core/tx/datashard/datashard_schema_snapshots.cpp +++ b/ydb/core/tx/datashard/datashard_schema_snapshots.cpp @@ -1,143 +1,143 @@ -#include "datashard_impl.h" -#include "datashard_schema_snapshots.h" - -#include <ydb/core/util/pb.h> - -namespace NKikimr { -namespace NDataShard { - -TSchemaSnapshot::TSchemaSnapshot(TUserTable::TCPtr schema, ui64 step, ui64 txId) - : Schema(schema) - , Step(step) - , TxId(txId) -{ -} - -TSchemaSnapshotManager::TSchemaSnapshotManager(const TDataShard* self) - : Self(self) -{ -} - -void TSchemaSnapshotManager::Reset() { - Snapshots.clear(); -} - -bool TSchemaSnapshotManager::Load(NIceDb::TNiceDb& db) { - using Schema = TDataShard::Schema; - - auto rowset = db.Table<Schema::SchemaSnapshots>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - const auto& tables = Self->GetUserTables(); - while (!rowset.EndOfSet()) { - const ui64 oid = rowset.GetValue<Schema::SchemaSnapshots::PathOwnerId>(); - const ui64 tid = rowset.GetValue<Schema::SchemaSnapshots::LocalPathId>(); - const ui64 version = rowset.GetValue<Schema::SchemaSnapshots::SchemaVersion>(); - const ui64 step = rowset.GetValue<Schema::SchemaSnapshots::Step>(); - const ui64 txId = rowset.GetValue<Schema::SchemaSnapshots::TxId>(); - const TString schema = rowset.GetValue<Schema::SchemaSnapshots::Schema>(); - - NKikimrSchemeOp::TTableDescription desc; - const bool ok = ParseFromStringNoSizeLimit(desc, schema); - Y_VERIFY(ok); - - auto it = tables.find(tid); - Y_VERIFY_S(it != tables.end(), "Cannot find table: " << tid); - - const auto res = Snapshots.emplace( - std::piecewise_construct, - std::forward_as_tuple(oid, tid, version), - std::forward_as_tuple(new TUserTable(it->second->LocalTid, desc, 0), step, txId) - ); - Y_VERIFY_S(res.second, "Duplicate schema snapshot: " << res.first->first); - - if (!rowset.Next()) { - return false; - } - } - - return true; -} - -bool TSchemaSnapshotManager::AddSnapshot(NTable::TDatabase& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot) { - if (auto it = Snapshots.find(key); it != Snapshots.end()) { - Y_VERIFY_DEBUG_S(false, "Duplicate schema snapshot: " << key); - return false; - } - - auto it = Self->GetUserTables().find(key.PathId); - Y_VERIFY_S(it != Self->GetUserTables().end(), "Cannot find table: " << key.PathId); - - const auto res = Snapshots.emplace(key, snapshot); - Y_VERIFY_S(res.second, "Duplicate schema snapshot: " << key); - - NIceDb::TNiceDb nicedb(db); - PersistAddSnapshot(nicedb, key, snapshot); - - return true; -} - -const TSchemaSnapshot* TSchemaSnapshotManager::FindSnapshot(const TSchemaSnapshotKey& key) const { - return Snapshots.FindPtr(key); -} - -bool TSchemaSnapshotManager::AcquireReference(const TSchemaSnapshotKey& key) { - auto it = Snapshots.find(key); - if (it == Snapshots.end()) { - return false; - } - - ++References[key]; - return true; -} - -bool TSchemaSnapshotManager::ReleaseReference(const TSchemaSnapshotKey& key, NIceDb::TNiceDb& db) { - auto refIt = References.find(key); - - if (refIt == References.end() || refIt->second <= 0) { - Y_VERIFY_DEBUG(false, "ReleaseReference underflow, check acquire/release pairs"); - return false; - } - - if (--refIt->second) { - return false; - } - - References.erase(refIt); - - auto it = Snapshots.find(key); - if (it == Snapshots.end()) { - Y_VERIFY_DEBUG(false, "ReleaseReference on an already removed snapshot"); - return false; - } - - PersistRemoveSnapshot(db, key); - return true; -} - -void TSchemaSnapshotManager::PersistAddSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot) { - using Schema = TDataShard::Schema; - db.Table<Schema::SchemaSnapshots>() - .Key(key.OwnerId, key.PathId, key.Version) - .Update( - NIceDb::TUpdate<Schema::SchemaSnapshots::Step>(snapshot.Step), - NIceDb::TUpdate<Schema::SchemaSnapshots::TxId>(snapshot.TxId), - NIceDb::TUpdate<Schema::SchemaSnapshots::Schema>(snapshot.Schema->GetSchema()) - ); -} - -void TSchemaSnapshotManager::PersistRemoveSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key) { - using Schema = TDataShard::Schema; - db.Table<Schema::SchemaSnapshots>() - .Key(key.OwnerId, key.PathId, key.Version) - .Delete(); -} - -} // NDataShard -} // NKikimr - -Y_DECLARE_OUT_SPEC(, NKikimr::NDataShard::TSchemaSnapshotKey, stream, value) { - stream << "{ table " << value.OwnerId << ":" << value.PathId << " version " << value.Version << " }"; -} +#include "datashard_impl.h" +#include "datashard_schema_snapshots.h" + +#include <ydb/core/util/pb.h> + +namespace NKikimr { +namespace NDataShard { + +TSchemaSnapshot::TSchemaSnapshot(TUserTable::TCPtr schema, ui64 step, ui64 txId) + : Schema(schema) + , Step(step) + , TxId(txId) +{ +} + +TSchemaSnapshotManager::TSchemaSnapshotManager(const TDataShard* self) + : Self(self) +{ +} + +void TSchemaSnapshotManager::Reset() { + Snapshots.clear(); +} + +bool TSchemaSnapshotManager::Load(NIceDb::TNiceDb& db) { + using Schema = TDataShard::Schema; + + auto rowset = db.Table<Schema::SchemaSnapshots>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + const auto& tables = Self->GetUserTables(); + while (!rowset.EndOfSet()) { + const ui64 oid = rowset.GetValue<Schema::SchemaSnapshots::PathOwnerId>(); + const ui64 tid = rowset.GetValue<Schema::SchemaSnapshots::LocalPathId>(); + const ui64 version = rowset.GetValue<Schema::SchemaSnapshots::SchemaVersion>(); + const ui64 step = rowset.GetValue<Schema::SchemaSnapshots::Step>(); + const ui64 txId = rowset.GetValue<Schema::SchemaSnapshots::TxId>(); + const TString schema = rowset.GetValue<Schema::SchemaSnapshots::Schema>(); + + NKikimrSchemeOp::TTableDescription desc; + const bool ok = ParseFromStringNoSizeLimit(desc, schema); + Y_VERIFY(ok); + + auto it = tables.find(tid); + Y_VERIFY_S(it != tables.end(), "Cannot find table: " << tid); + + const auto res = Snapshots.emplace( + std::piecewise_construct, + std::forward_as_tuple(oid, tid, version), + std::forward_as_tuple(new TUserTable(it->second->LocalTid, desc, 0), step, txId) + ); + Y_VERIFY_S(res.second, "Duplicate schema snapshot: " << res.first->first); + + if (!rowset.Next()) { + return false; + } + } + + return true; +} + +bool TSchemaSnapshotManager::AddSnapshot(NTable::TDatabase& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot) { + if (auto it = Snapshots.find(key); it != Snapshots.end()) { + Y_VERIFY_DEBUG_S(false, "Duplicate schema snapshot: " << key); + return false; + } + + auto it = Self->GetUserTables().find(key.PathId); + Y_VERIFY_S(it != Self->GetUserTables().end(), "Cannot find table: " << key.PathId); + + const auto res = Snapshots.emplace(key, snapshot); + Y_VERIFY_S(res.second, "Duplicate schema snapshot: " << key); + + NIceDb::TNiceDb nicedb(db); + PersistAddSnapshot(nicedb, key, snapshot); + + return true; +} + +const TSchemaSnapshot* TSchemaSnapshotManager::FindSnapshot(const TSchemaSnapshotKey& key) const { + return Snapshots.FindPtr(key); +} + +bool TSchemaSnapshotManager::AcquireReference(const TSchemaSnapshotKey& key) { + auto it = Snapshots.find(key); + if (it == Snapshots.end()) { + return false; + } + + ++References[key]; + return true; +} + +bool TSchemaSnapshotManager::ReleaseReference(const TSchemaSnapshotKey& key, NIceDb::TNiceDb& db) { + auto refIt = References.find(key); + + if (refIt == References.end() || refIt->second <= 0) { + Y_VERIFY_DEBUG(false, "ReleaseReference underflow, check acquire/release pairs"); + return false; + } + + if (--refIt->second) { + return false; + } + + References.erase(refIt); + + auto it = Snapshots.find(key); + if (it == Snapshots.end()) { + Y_VERIFY_DEBUG(false, "ReleaseReference on an already removed snapshot"); + return false; + } + + PersistRemoveSnapshot(db, key); + return true; +} + +void TSchemaSnapshotManager::PersistAddSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot) { + using Schema = TDataShard::Schema; + db.Table<Schema::SchemaSnapshots>() + .Key(key.OwnerId, key.PathId, key.Version) + .Update( + NIceDb::TUpdate<Schema::SchemaSnapshots::Step>(snapshot.Step), + NIceDb::TUpdate<Schema::SchemaSnapshots::TxId>(snapshot.TxId), + NIceDb::TUpdate<Schema::SchemaSnapshots::Schema>(snapshot.Schema->GetSchema()) + ); +} + +void TSchemaSnapshotManager::PersistRemoveSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key) { + using Schema = TDataShard::Schema; + db.Table<Schema::SchemaSnapshots>() + .Key(key.OwnerId, key.PathId, key.Version) + .Delete(); +} + +} // NDataShard +} // NKikimr + +Y_DECLARE_OUT_SPEC(, NKikimr::NDataShard::TSchemaSnapshotKey, stream, value) { + stream << "{ table " << value.OwnerId << ":" << value.PathId << " version " << value.Version << " }"; +} diff --git a/ydb/core/tx/datashard/datashard_schema_snapshots.h b/ydb/core/tx/datashard/datashard_schema_snapshots.h index 28ce7a3f965..a8218a22591 100644 --- a/ydb/core/tx/datashard/datashard_schema_snapshots.h +++ b/ydb/core/tx/datashard/datashard_schema_snapshots.h @@ -1,50 +1,50 @@ -#pragma once - -#include "datashard_user_table.h" -#include "snapshot_key.h" - -#include <ydb/core/tablet_flat/flat_cxx_database.h> - -#include <util/generic/hash.h> -#include <util/generic/map.h> - -namespace NKikimr { -namespace NDataShard { - -class TDataShard; -class TSchemaSnapshotManager; - -struct TSchemaSnapshot { - TUserTable::TCPtr Schema; - ui64 Step = 0; - ui64 TxId = 0; - - explicit TSchemaSnapshot(TUserTable::TCPtr schema, ui64 step, ui64 txId); -}; - -class TSchemaSnapshotManager { -public: - explicit TSchemaSnapshotManager(const TDataShard* self); - - void Reset(); - bool Load(NIceDb::TNiceDb& db); - - bool AddSnapshot(NTable::TDatabase& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot); - const TSchemaSnapshot* FindSnapshot(const TSchemaSnapshotKey& key) const; - - bool AcquireReference(const TSchemaSnapshotKey& key); - bool ReleaseReference(const TSchemaSnapshotKey& key, NIceDb::TNiceDb& db); - -private: - void PersistAddSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot); - void PersistRemoveSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key); - -private: - const TDataShard* Self; - TMap<TSchemaSnapshotKey, TSchemaSnapshot, TLess<void>> Snapshots; - THashMap<TSchemaSnapshotKey, size_t> References; - -}; // TSchemaSnapshotManager - -} // NDataShard -} // NKikimr +#pragma once + +#include "datashard_user_table.h" +#include "snapshot_key.h" + +#include <ydb/core/tablet_flat/flat_cxx_database.h> + +#include <util/generic/hash.h> +#include <util/generic/map.h> + +namespace NKikimr { +namespace NDataShard { + +class TDataShard; +class TSchemaSnapshotManager; + +struct TSchemaSnapshot { + TUserTable::TCPtr Schema; + ui64 Step = 0; + ui64 TxId = 0; + + explicit TSchemaSnapshot(TUserTable::TCPtr schema, ui64 step, ui64 txId); +}; + +class TSchemaSnapshotManager { +public: + explicit TSchemaSnapshotManager(const TDataShard* self); + + void Reset(); + bool Load(NIceDb::TNiceDb& db); + + bool AddSnapshot(NTable::TDatabase& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot); + const TSchemaSnapshot* FindSnapshot(const TSchemaSnapshotKey& key) const; + + bool AcquireReference(const TSchemaSnapshotKey& key); + bool ReleaseReference(const TSchemaSnapshotKey& key, NIceDb::TNiceDb& db); + +private: + void PersistAddSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key, const TSchemaSnapshot& snapshot); + void PersistRemoveSnapshot(NIceDb::TNiceDb& db, const TSchemaSnapshotKey& key); + +private: + const TDataShard* Self; + TMap<TSchemaSnapshotKey, TSchemaSnapshot, TLess<void>> Snapshots; + THashMap<TSchemaSnapshotKey, size_t> References; + +}; // TSchemaSnapshotManager + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_snapshots.cpp b/ydb/core/tx/datashard/datashard_snapshots.cpp index 5641de37fd1..da1219bb9e6 100644 --- a/ydb/core/tx/datashard/datashard_snapshots.cpp +++ b/ydb/core/tx/datashard/datashard_snapshots.cpp @@ -147,15 +147,15 @@ void TSnapshotManager::SetCompleteEdge(NIceDb::TNiceDb& db, const TRowVersion& v CompleteEdge = version; } -bool TSnapshotManager::PromoteCompleteEdge(const TRowVersion& version, TTransactionContext& txc) { +bool TSnapshotManager::PromoteCompleteEdge(const TRowVersion& version, TTransactionContext& txc) { if (!IsMvccEnabled()) return false; - if (CompleteEdge >= version) + if (CompleteEdge >= version) return false; NIceDb::TNiceDb db(txc.DB); - SetCompleteEdge(db, version); + SetCompleteEdge(db, version); txc.OnCommitted([this, edge = CompleteEdge] { this->CommittedCompleteEdge = edge; }); @@ -163,23 +163,23 @@ bool TSnapshotManager::PromoteCompleteEdge(const TRowVersion& version, TTransact return true; } -bool TSnapshotManager::PromoteCompleteEdge(TOperation* op, TTransactionContext& txc) { - if (!IsMvccEnabled()) - return false; - - Y_ASSERT(op && (op->IsMvccSnapshotRead() || op->GetStep())); - - const TRowVersion version = op->IsMvccSnapshotRead() - ? op->GetMvccSnapshot() - : TRowVersion(op->GetStep(), op->GetTxId()); - - return PromoteCompleteEdge(version, txc); -} - -bool TSnapshotManager::PromoteCompleteEdge(ui64 writeStep, TTransactionContext& txc) { - return PromoteCompleteEdge(TRowVersion(writeStep, 0), txc); -} - +bool TSnapshotManager::PromoteCompleteEdge(TOperation* op, TTransactionContext& txc) { + if (!IsMvccEnabled()) + return false; + + Y_ASSERT(op && (op->IsMvccSnapshotRead() || op->GetStep())); + + const TRowVersion version = op->IsMvccSnapshotRead() + ? op->GetMvccSnapshot() + : TRowVersion(op->GetStep(), op->GetTxId()); + + return PromoteCompleteEdge(version, txc); +} + +bool TSnapshotManager::PromoteCompleteEdge(ui64 writeStep, TTransactionContext& txc) { + return PromoteCompleteEdge(TRowVersion(writeStep, 0), txc); +} + TRowVersion TSnapshotManager::GetIncompleteEdge() const { return IncompleteEdge; } diff --git a/ydb/core/tx/datashard/datashard_snapshots.h b/ydb/core/tx/datashard/datashard_snapshots.h index 3c6b49d331c..87468341120 100644 --- a/ydb/core/tx/datashard/datashard_snapshots.h +++ b/ydb/core/tx/datashard/datashard_snapshots.h @@ -1,7 +1,7 @@ #pragma once #include "const.h" -#include "snapshot_key.h" +#include "snapshot_key.h" #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/tablet_flat/tablet_flat_executor.h> @@ -121,7 +121,7 @@ public: void SetCompleteEdge(NIceDb::TNiceDb& db, const TRowVersion& version); - bool PromoteCompleteEdge(const TRowVersion& version, TTransactionContext& txc); + bool PromoteCompleteEdge(const TRowVersion& version, TTransactionContext& txc); bool PromoteCompleteEdge(TOperation* op, TTransactionContext& txc); bool PromoteCompleteEdge(ui64 writeStep, TTransactionContext& txc); diff --git a/ydb/core/tx/datashard/datashard_split_dst.cpp b/ydb/core/tx/datashard/datashard_split_dst.cpp index d6ed124315a..65750810013 100644 --- a/ydb/core/tx/datashard/datashard_split_dst.cpp +++ b/ydb/core/tx/datashard/datashard_split_dst.cpp @@ -234,11 +234,11 @@ public: db.Table<Schema::SplitDstReceivedSnapshots>().Key(srcTabletId).Update(); Self->ReceiveSnapshotsFrom.erase(srcTabletId); - if (record.GetWaitForActivation()) { - Self->ReceiveActivationsFrom.insert(srcTabletId); - db.Table<Schema::DstChangeSenderActivations>().Key(srcTabletId).Update(); - } - + if (record.GetWaitForActivation()) { + Self->ReceiveActivationsFrom.insert(srcTabletId); + db.Table<Schema::DstChangeSenderActivations>().Key(srcTabletId).Update(); + } + if (Self->ReceiveSnapshotsFrom.empty()) { LastSnapshotReceived = true; diff --git a/ydb/core/tx/datashard/datashard_split_src.cpp b/ydb/core/tx/datashard/datashard_split_src.cpp index e5bf204f695..a7349fc9cf3 100644 --- a/ydb/core/tx/datashard/datashard_split_src.cpp +++ b/ydb/core/tx/datashard/datashard_split_src.cpp @@ -197,7 +197,7 @@ public: return true; } - void Complete(const TActorContext &) override { + void Complete(const TActorContext &) override { } }; @@ -209,13 +209,13 @@ NTabletFlatExecutor::ITransaction* TDataShard::CreateTxStartSplit() { class TDataShard::TTxSplitSnapshotComplete : public NTabletFlatExecutor::TTransactionBase<TDataShard> { private: TIntrusivePtr<TSplitSnapshotContext> SnapContext; - bool ChangeExchangeSplit; + bool ChangeExchangeSplit; public: TTxSplitSnapshotComplete(TDataShard* ds, TIntrusivePtr<TSplitSnapshotContext> snapContext) : NTabletFlatExecutor::TTransactionBase<TDataShard>(ds) , SnapContext(snapContext) - , ChangeExchangeSplit(false) + , ChangeExchangeSplit(false) {} TTxType GetTxType() const override { return TXTYPE_SPLIT_SNASHOT_COMPLETE; } @@ -341,16 +341,16 @@ public: proto->SetTimeoutMs(kv.second.Timeout.MilliSeconds()); } - if (Self->ChangesQueue || tableInfo.HasCdcStreams()) { - snapshot->SetWaitForActivation(true); - Self->ChangeSenderActivator.AddDst(dstTablet); - db.Table<Schema::SrcChangeSenderActivations>().Key(dstTablet).Update(); - - if (tableInfo.HasCdcStreams()) { - Self->ChangeExchangeSplitter.AddDst(dstTablet); - } - } - + if (Self->ChangesQueue || tableInfo.HasCdcStreams()) { + snapshot->SetWaitForActivation(true); + Self->ChangeSenderActivator.AddDst(dstTablet); + db.Table<Schema::SrcChangeSenderActivations>().Key(dstTablet).Update(); + + if (tableInfo.HasCdcStreams()) { + Self->ChangeExchangeSplitter.AddDst(dstTablet); + } + } + if (sourceOffsetsBytes > 0) { snapshot->SetReplicationSourceOffsetsBytes(sourceOffsetsBytes); } @@ -366,8 +366,8 @@ public: } } - ChangeExchangeSplit = !Self->ChangesQueue && !Self->ChangeExchangeSplitter.Done(); - + ChangeExchangeSplit = !Self->ChangesQueue && !Self->ChangeExchangeSplitter.Done(); + if (needToReadPages) { LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " BorrowSnapshot is restarting for split OpId " << opId); return false; @@ -384,9 +384,9 @@ public: void Complete(const TActorContext &ctx) override { LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " Sending snapshots from src for split OpId " << Self->SrcSplitOpId); Self->SplitSrcSnapshotSender.DoSend(ctx); - if (ChangeExchangeSplit) { - Self->ChangeExchangeSplitter.DoSplit(ctx); - } + if (ChangeExchangeSplit) { + Self->ChangeExchangeSplitter.DoSplit(ctx); + } } }; @@ -400,14 +400,14 @@ class TDataShard::TTxSplitTransferSnapshotAck : public NTabletFlatExecutor::TTra private: TEvDataShard::TEvSplitTransferSnapshotAck::TPtr Ev; bool AllDstAcksReceived; - bool Activate; + bool Activate; public: TTxSplitTransferSnapshotAck(TDataShard* ds, TEvDataShard::TEvSplitTransferSnapshotAck::TPtr& ev) : NTabletFlatExecutor::TTransactionBase<TDataShard>(ds) , Ev(ev) , AllDstAcksReceived(false) - , Activate(false) + , Activate(false) {} TTxType GetTxType() const override { return TXTYPE_SPLIT_TRANSFER_SNAPSHOT_ACK; } @@ -431,10 +431,10 @@ public: // Remove the row for acked snapshot db.Table<Schema::SplitSrcSnapshots>().Key(dstTabletId).Delete(); - if (!Self->ChangesQueue && Self->ChangeExchangeSplitter.Done()) { - Activate = !Self->ChangeSenderActivator.Acked(dstTabletId); - } - + if (!Self->ChangesQueue && Self->ChangeExchangeSplitter.Done()) { + Activate = !Self->ChangeSenderActivator.Acked(dstTabletId); + } + return true; } @@ -446,13 +446,13 @@ public: ctx.Send(ackTo, new TEvDataShard::TEvSplitAck(opId, Self->TabletID())); } } - - if (Activate) { - const ui64 dstTabletId = Ev->Get()->Record.GetTabletId(); - if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { - Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); - } - } + + if (Activate) { + const ui64 dstTabletId = Ev->Get()->Record.GetTabletId(); + if (!Self->ChangeSenderActivator.Acked(dstTabletId)) { + Self->ChangeSenderActivator.DoSend(dstTabletId, ctx); + } + } } }; @@ -460,7 +460,7 @@ public: class TDataShard::TTxSplitPartitioningChanged : public NTabletFlatExecutor::TTransactionBase<TDataShard> { private: TEvDataShard::TEvSplitPartitioningChanged::TPtr Ev; - bool DelayPartitioningChangedAck = false; + bool DelayPartitioningChangedAck = false; public: TTxSplitPartitioningChanged(TDataShard* ds, TEvDataShard::TEvSplitPartitioningChanged::TPtr& ev) @@ -471,20 +471,20 @@ public: TTxType GetTxType() const override { return TXTYPE_SPLIT_PARTITIONING_CHANGED; } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - ui64 opId = Ev->Get()->Record.GetOperationCookie(); - + ui64 opId = Ev->Get()->Record.GetOperationCookie(); + LOG_DEBUG(ctx, NKikimrServices::TX_DATASHARD, "Got TEvSplitPartitioningChanged opId %" PRIu64 " at datashard %" PRIu64 " state %s", - opId, Self->TabletID(), DatashardStateName(Self->State).data()); - - if (Self->ChangesQueue || !Self->ChangeSenderActivator.AllAcked()) { - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " delay partitioning changed ack" - << " ChangesQueue size: " << Self->ChangesQueue.size() - << " siblings to be activated: " << Self->ChangeSenderActivator.Dump()); - - DelayPartitioningChangedAck = true; - Self->SrcAckPartitioningChangedTo[Ev->Sender].insert(opId); - } - + opId, Self->TabletID(), DatashardStateName(Self->State).data()); + + if (Self->ChangesQueue || !Self->ChangeSenderActivator.AllAcked()) { + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " delay partitioning changed ack" + << " ChangesQueue size: " << Self->ChangesQueue.size() + << " siblings to be activated: " << Self->ChangeSenderActivator.Dump()); + + DelayPartitioningChangedAck = true; + Self->SrcAckPartitioningChangedTo[Ev->Sender].insert(opId); + } + // TODO: At this point Src should start rejecting all new Tx with SchemaChanged status if (Self->State != TShardState::SplitSrcWaitForPartitioningChanged) { Y_VERIFY(Self->State == TShardState::PreOffline || Self->State == TShardState::Offline, @@ -507,10 +507,10 @@ public: TActorId ackTo = Ev->Sender; ui64 opId = Ev->Get()->Record.GetOperationCookie(); - if (DelayPartitioningChangedAck) { - return; - } - + if (DelayPartitioningChangedAck) { + return; + } + LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, Self->TabletID() << " ack split partitioning changed to schemeshard " << opId); ctx.Send(ackTo, new TEvDataShard::TEvSplitPartitioningChangedAck(opId, Self->TabletID())); diff --git a/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp b/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp index ca048ef2372..04c09cf74bf 100644 --- a/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp +++ b/ydb/core/tx/datashard/datashard_switch_mvcc_state.cpp @@ -13,7 +13,7 @@ void TDataShard::CheckMvccStateChangeCanStart(const TActorContext& ctx) { case TShardState::Ready: case TShardState::Frozen: { - const auto enable = AppData(ctx)->FeatureFlags.GetEnableMvcc(); + const auto enable = AppData(ctx)->FeatureFlags.GetEnableMvcc(); if (enable && *enable != IsMvccEnabled()) { MvccSwitchState = TSwitchState::SWITCHING; } else { @@ -79,10 +79,10 @@ bool TDataShard::TTxExecuteMvccStateChange::Execute(TTransactionContext& txc, co auto [step, txId] = Self->LastCompleteTxVersion(); Self->SnapshotManager.ChangeMvccState(step, txId, txc, - *AppData(ctx)->FeatureFlags.GetEnableMvcc() ? EMvccState::MvccEnabled : EMvccState::MvccDisabled); + *AppData(ctx)->FeatureFlags.GetEnableMvcc() ? EMvccState::MvccEnabled : EMvccState::MvccDisabled); LOG_DEBUG(ctx, NKikimrServices::TX_DATASHARD, TStringBuilder() << "TTxExecuteMvccStateChange.Execute" - << " MVCC state switched to" << (*AppData(ctx)->FeatureFlags.GetEnableMvcc() ? " enabled" : " disabled") << " state"); + << " MVCC state switched to" << (*AppData(ctx)->FeatureFlags.GetEnableMvcc() ? " enabled" : " disabled") << " state"); ActivateWaitingOps = true; } diff --git a/ydb/core/tx/datashard/datashard_trans_queue.cpp b/ydb/core/tx/datashard/datashard_trans_queue.cpp index 607b6fdd41e..d7096c25b32 100644 --- a/ydb/core/tx/datashard/datashard_trans_queue.cpp +++ b/ydb/core/tx/datashard/datashard_trans_queue.cpp @@ -131,21 +131,21 @@ bool TTransQueue::Load(NIceDb::TNiceDb& db) { ui64 maxStep = rowset.GetValue<Schema::SchemaOperations::MaxStep>(); ui64 planStep = rowset.GetValueOrDefault<Schema::SchemaOperations::PlanStep>(0); ui64 readOnly = rowset.GetValueOrDefault<Schema::SchemaOperations::ReadOnly>(false); - bool success = rowset.GetValueOrDefault<Schema::SchemaOperations::Success>(false); - TString error = rowset.GetValueOrDefault<Schema::SchemaOperations::Error>(TString()); - ui64 dataSize = rowset.GetValueOrDefault<Schema::SchemaOperations::DataSize>(0); - ui64 rows = rowset.GetValueOrDefault<Schema::SchemaOperations::Rows>(0); + bool success = rowset.GetValueOrDefault<Schema::SchemaOperations::Success>(false); + TString error = rowset.GetValueOrDefault<Schema::SchemaOperations::Error>(TString()); + ui64 dataSize = rowset.GetValueOrDefault<Schema::SchemaOperations::DataSize>(0); + ui64 rows = rowset.GetValueOrDefault<Schema::SchemaOperations::Rows>(0); if (!tabletId) { // Remove legacy data from DB. New schema operations has tabletId db.Table<Schema::SchemaOperations>().Key(txId).Delete(); continue; } - TSchemaOperation op(txId, TSchemaOperation::EType(opType), source, - tabletId, minStep, maxStep, planStep, readOnly, - success, error, dataSize, rows); + TSchemaOperation op(txId, TSchemaOperation::EType(opType), source, + tabletId, minStep, maxStep, planStep, readOnly, + success, error, dataSize, rows); auto saved = SchemaOps.insert(std::make_pair(op.TxId, op)); - TSchemaOperation * savedOp = &saved.first->second; + TSchemaOperation * savedOp = &saved.first->second; if (schemaTxs.contains(txId)) { // is not done yet Self->Pipeline.SetSchemaOp(savedOp); } else { @@ -162,7 +162,7 @@ bool TTransQueue::Load(NIceDb::TNiceDb& db) { return false; while (!rowset.EndOfSet()) { ui64 txId = rowset.GetValue<Schema::ScanProgress::TxId>(); - TSchemaOperation* op = FindSchemaTx(txId); + TSchemaOperation* op = FindSchemaTx(txId); if (!op) { LOG_WARN_S(TlsActivationContext->AsActorContext(), NKikimrServices::TX_DATASHARD, "Op was not found for persisted scan tx id " << txId @@ -185,7 +185,7 @@ bool TTransQueue::Load(NIceDb::TNiceDb& db) { return true; } -void TTransQueue::ProposeSchemaTx(NIceDb::TNiceDb& db, const TSchemaOperation& op) { +void TTransQueue::ProposeSchemaTx(NIceDb::TNiceDb& db, const TSchemaOperation& op) { using Schema = TDataShard::Schema; // Auto-ack previous schema operation @@ -206,13 +206,13 @@ void TTransQueue::ProposeSchemaTx(NIceDb::TNiceDb& db, const TSchemaOperation& o NIceDb::TUpdate<Schema::SchemaOperations::MinStep>(op.MinStep), NIceDb::TUpdate<Schema::SchemaOperations::MaxStep>(op.MaxStep), NIceDb::TUpdate<Schema::SchemaOperations::PlanStep>(op.PlanStep), - NIceDb::TUpdate<Schema::SchemaOperations::ReadOnly>(op.ReadOnly), - NIceDb::TUpdate<Schema::SchemaOperations::Success>(op.Success), - NIceDb::TUpdate<Schema::SchemaOperations::Error>(op.Error), - NIceDb::TUpdate<Schema::SchemaOperations::DataSize>(op.BytesProcessed), - NIceDb::TUpdate<Schema::SchemaOperations::Rows>(op.RowsProcessed)); + NIceDb::TUpdate<Schema::SchemaOperations::ReadOnly>(op.ReadOnly), + NIceDb::TUpdate<Schema::SchemaOperations::Success>(op.Success), + NIceDb::TUpdate<Schema::SchemaOperations::Error>(op.Error), + NIceDb::TUpdate<Schema::SchemaOperations::DataSize>(op.BytesProcessed), + NIceDb::TUpdate<Schema::SchemaOperations::Rows>(op.RowsProcessed)); - TSchemaOperation * savedOp = &saved.first->second; + TSchemaOperation * savedOp = &saved.first->second; Y_VERIFY(savedOp->TabletId); Self->Pipeline.SetSchemaOp(savedOp); diff --git a/ydb/core/tx/datashard/datashard_trans_queue.h b/ydb/core/tx/datashard/datashard_trans_queue.h index 8e5201c1901..1d392676da3 100644 --- a/ydb/core/tx/datashard/datashard_trans_queue.h +++ b/ydb/core/tx/datashard/datashard_trans_queue.h @@ -65,8 +65,8 @@ public: return EMPTY_PLAN; } - TSchemaOperation * FindSchemaTx(ui64 txId) { return SchemaOps.FindPtr(txId); } - const TMap<ui64, TSchemaOperation>& GetSchemaOperations() const { return SchemaOps; } + TSchemaOperation * FindSchemaTx(ui64 txId) { return SchemaOps.FindPtr(txId); } + const TMap<ui64, TSchemaOperation>& GetSchemaOperations() const { return SchemaOps; } bool HasNotAckedSchemaTx() const { return ! SchemaOps.empty(); } ui64 TxPlanWaiting() const { return PlanWaitingTxCount; } @@ -84,7 +84,7 @@ private: // for pipeline only void ProposeTx(NIceDb::TNiceDb& db, TOperation::TPtr op, TActorId source, const TStringBuf& txBody); void UpdateTxFlags(NIceDb::TNiceDb& db, ui64 txId, ui64 flags); void UpdateTxBody(NIceDb::TNiceDb& db, ui64 txId, const TStringBuf& txBody); - void ProposeSchemaTx(NIceDb::TNiceDb& db, const TSchemaOperation& op); + void ProposeSchemaTx(NIceDb::TNiceDb& db, const TSchemaOperation& op); bool CancelPropose(NIceDb::TNiceDb& db, ui64 txId); ECleanupStatus CleanupOutdated(NIceDb::TNiceDb& db, ui64 outdatedStep, ui32 batchSize, TVector<ui64>& outdatedTxs); @@ -126,7 +126,7 @@ private: TSet<TStepOrder> PlannedTxs; THashMap<EOperationKind, TSet<TStepOrder>> PlannedTxsByKind; TSet<std::pair<ui64, ui64>> DeadlineQueue; // {maxStep, txId} - TMap<ui64, TSchemaOperation> SchemaOps; // key - txId + TMap<ui64, TSchemaOperation> SchemaOps; // key - txId TSet<ui64> ProposeDelayers; ui64 PlanWaitingTxCount = 0; diff --git a/ydb/core/tx/datashard/datashard_txs.h b/ydb/core/tx/datashard/datashard_txs.h index b2638a8c2ca..4a9ab008789 100644 --- a/ydb/core/tx/datashard/datashard_txs.h +++ b/ydb/core/tx/datashard/datashard_txs.h @@ -1,6 +1,6 @@ #pragma once -#include "datashard_common_upload.h" +#include "datashard_common_upload.h" #include "datashard_impl.h" #include "execution_unit_kind.h" @@ -46,8 +46,8 @@ public: private: bool CreateScheme(TTransactionContext &txc); bool ReadEverything(TTransactionContext &txc); -private: - TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo> ChangeRecords; +private: + TVector<TEvChangeExchange::TEvEnqueueRecords::TRecordInfo> ChangeRecords; }; class TDataShard::TTxPlanStep : public NTabletFlatExecutor::TTransactionBase<TDataShard> { @@ -219,71 +219,71 @@ private: }; class TDataShard::TTxGetS3Upload: public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxGetS3Upload(TDataShard* ds, TEvDataShard::TEvGetS3Upload::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_GET_S3_UPLOAD_ID; } -private: - TEvDataShard::TEvGetS3Upload::TPtr Ev; - THolder<TEvDataShard::TEvS3Upload> Reply; -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_GET_S3_UPLOAD_ID; } +private: + TEvDataShard::TEvGetS3Upload::TPtr Ev; + THolder<TEvDataShard::TEvS3Upload> Reply; +}; + class TDataShard::TTxStoreS3UploadId: public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxStoreS3UploadId(TDataShard* ds, TEvDataShard::TEvStoreS3UploadId::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_STORE_S3_UPLOAD_ID; } -private: - TEvDataShard::TEvStoreS3UploadId::TPtr Ev; - THolder<TEvDataShard::TEvS3Upload> Reply; -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_STORE_S3_UPLOAD_ID; } +private: + TEvDataShard::TEvStoreS3UploadId::TPtr Ev; + THolder<TEvDataShard::TEvS3Upload> Reply; +}; + class TDataShard::TTxChangeS3UploadStatus: public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxChangeS3UploadStatus(TDataShard* ds, TEvDataShard::TEvChangeS3UploadStatus::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_CHANGE_S3_UPLOAD_STATUS; } -private: - TEvDataShard::TEvChangeS3UploadStatus::TPtr Ev; - THolder<TEvDataShard::TEvS3Upload> Reply; -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_CHANGE_S3_UPLOAD_STATUS; } +private: + TEvDataShard::TEvChangeS3UploadStatus::TPtr Ev; + THolder<TEvDataShard::TEvS3Upload> Reply; +}; + class TDataShard::TTxGetS3DownloadInfo: public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxGetS3DownloadInfo(TDataShard* ds, TEvDataShard::TEvGetS3DownloadInfo::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_GET_S3_DOWNLOAD_INFO; } -private: - TEvDataShard::TEvGetS3DownloadInfo::TPtr Ev; - THolder<TEvDataShard::TEvS3DownloadInfo> Reply; -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_GET_S3_DOWNLOAD_INFO; } +private: + TEvDataShard::TEvGetS3DownloadInfo::TPtr Ev; + THolder<TEvDataShard::TEvS3DownloadInfo> Reply; +}; + class TDataShard::TTxStoreS3DownloadInfo: public NTabletFlatExecutor::TTransactionBase<TDataShard> { -public: +public: TTxStoreS3DownloadInfo(TDataShard* ds, TEvDataShard::TEvStoreS3DownloadInfo::TPtr ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_STORE_S3_DOWNLOAD_INFO; } -private: - TEvDataShard::TEvStoreS3DownloadInfo::TPtr Ev; - THolder<TEvDataShard::TEvS3DownloadInfo> Reply; -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_STORE_S3_DOWNLOAD_INFO; } +private: + TEvDataShard::TEvStoreS3DownloadInfo::TPtr Ev; + THolder<TEvDataShard::TEvS3DownloadInfo> Reply; +}; + class TDataShard::TTxUnsafeUploadRows: public NTabletFlatExecutor::TTransactionBase<TDataShard>, - public TCommonUploadOps< - TEvDataShard::TEvUnsafeUploadRowsRequest, - TEvDataShard::TEvUnsafeUploadRowsResponse> { -public: + public TCommonUploadOps< + TEvDataShard::TEvUnsafeUploadRowsRequest, + TEvDataShard::TEvUnsafeUploadRowsResponse> { +public: TTxUnsafeUploadRows(TDataShard* ds, TEvDataShard::TEvUnsafeUploadRowsRequest::TPtr& ev); - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_UNSAFE_UPLOAD_ROWS; } -}; - + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + TTxType GetTxType() const override { return TXTYPE_UNSAFE_UPLOAD_ROWS; } +}; + class TDataShard::TTxExecuteMvccStateChange: public NTabletFlatExecutor::TTransactionBase<TDataShard> { public: TTxExecuteMvccStateChange(TDataShard* ds); diff --git a/ydb/core/tx/datashard/datashard_unsafe_upload.cpp b/ydb/core/tx/datashard/datashard_unsafe_upload.cpp index abb5452e073..273952a06d1 100644 --- a/ydb/core/tx/datashard/datashard_unsafe_upload.cpp +++ b/ydb/core/tx/datashard/datashard_unsafe_upload.cpp @@ -1,26 +1,26 @@ -#include "datashard_txs.h" - -namespace NKikimr { +#include "datashard_txs.h" + +namespace NKikimr { namespace NDataShard { - + TDataShard::TTxUnsafeUploadRows::TTxUnsafeUploadRows(TDataShard* ds, TEvDataShard::TEvUnsafeUploadRowsRequest::TPtr& ev) - : TBase(ds) - , TCommonUploadOps(ev, false, false) -{ -} - + : TBase(ds) + , TCommonUploadOps(ev, false, false) +{ +} + bool TDataShard::TTxUnsafeUploadRows::Execute(TTransactionContext& txc, const TActorContext&) { auto [readVersion, writeVersion] = Self->GetReadWriteVersions(); - if (!TCommonUploadOps::Execute(Self, txc, readVersion, writeVersion)) + if (!TCommonUploadOps::Execute(Self, txc, readVersion, writeVersion)) return false; Self->PromoteCompleteEdge(writeVersion.Step, txc); return true; -} - +} + void TDataShard::TTxUnsafeUploadRows::Complete(const TActorContext& ctx) { - TCommonUploadOps::SendResult(Self, ctx); -} - + TCommonUploadOps::SendResult(Self, ctx); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index cd9b4c33297..70bc9400733 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -68,124 +68,124 @@ bool TUserTable::ResetTableSchemaVersion() } void TUserTable::AddIndex(const NKikimrSchemeOp::TIndexDescription& indexDesc) { - Y_VERIFY(indexDesc.HasPathOwnerId() && indexDesc.HasLocalPathId()); - const auto addIndexPathId = TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()); - - if (Indexes.contains(addIndexPathId)) { - return; - } - - Indexes.emplace(addIndexPathId, TTableIndex(indexDesc, Columns)); - AsyncIndexCount += ui32(indexDesc.GetType() == TTableIndex::EIndexType::EIndexTypeGlobalAsync); - + Y_VERIFY(indexDesc.HasPathOwnerId() && indexDesc.HasLocalPathId()); + const auto addIndexPathId = TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()); + + if (Indexes.contains(addIndexPathId)) { + return; + } + + Indexes.emplace(addIndexPathId, TTableIndex(indexDesc, Columns)); + AsyncIndexCount += ui32(indexDesc.GetType() == TTableIndex::EIndexType::EIndexTypeGlobalAsync); + NKikimrSchemeOp::TTableDescription schema; - GetSchema(schema); - - schema.MutableTableIndexes()->Add()->CopyFrom(indexDesc); - SetSchema(schema); -} - -void TUserTable::DropIndex(const TPathId& indexPathId) { - auto it = Indexes.find(indexPathId); - if (it == Indexes.end()) { - return; - } - - AsyncIndexCount -= ui32(it->second.Type == TTableIndex::EIndexType::EIndexTypeGlobalAsync); - Indexes.erase(it); - + GetSchema(schema); + + schema.MutableTableIndexes()->Add()->CopyFrom(indexDesc); + SetSchema(schema); +} + +void TUserTable::DropIndex(const TPathId& indexPathId) { + auto it = Indexes.find(indexPathId); + if (it == Indexes.end()) { + return; + } + + AsyncIndexCount -= ui32(it->second.Type == TTableIndex::EIndexType::EIndexTypeGlobalAsync); + Indexes.erase(it); + NKikimrSchemeOp::TTableDescription schema; - GetSchema(schema); - - for (auto it = schema.GetTableIndexes().begin(); it != schema.GetTableIndexes().end(); ++it) { - if (indexPathId != TPathId(it->GetPathOwnerId(), it->GetLocalPathId())) { - continue; - } - - schema.MutableTableIndexes()->erase(it); - SetSchema(schema); - - return; - } - - Y_FAIL("unreachable"); -} - -bool TUserTable::HasAsyncIndexes() const { - return AsyncIndexCount > 0; -} - + GetSchema(schema); + + for (auto it = schema.GetTableIndexes().begin(); it != schema.GetTableIndexes().end(); ++it) { + if (indexPathId != TPathId(it->GetPathOwnerId(), it->GetLocalPathId())) { + continue; + } + + schema.MutableTableIndexes()->erase(it); + SetSchema(schema); + + return; + } + + Y_FAIL("unreachable"); +} + +bool TUserTable::HasAsyncIndexes() const { + return AsyncIndexCount > 0; +} + void TUserTable::AddCdcStream(const NKikimrSchemeOp::TCdcStreamDescription& streamDesc) { - Y_VERIFY(streamDesc.HasPathId()); - const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); - - if (CdcStreams.contains(streamPathId)) { - return; - } - - CdcStreams.emplace(streamPathId, TCdcStream(streamDesc)); - + Y_VERIFY(streamDesc.HasPathId()); + const auto streamPathId = TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()); + + if (CdcStreams.contains(streamPathId)) { + return; + } + + CdcStreams.emplace(streamPathId, TCdcStream(streamDesc)); + NKikimrSchemeOp::TTableDescription schema; - GetSchema(schema); - - schema.MutableCdcStreams()->Add()->CopyFrom(streamDesc); - SetSchema(schema); -} - -void TUserTable::DisableCdcStream(const TPathId& streamPathId) { - auto it = CdcStreams.find(streamPathId); - if (it == CdcStreams.end()) { - return; - } - - it->second.State = TCdcStream::EState::ECdcStreamStateDisabled; - + GetSchema(schema); + + schema.MutableCdcStreams()->Add()->CopyFrom(streamDesc); + SetSchema(schema); +} + +void TUserTable::DisableCdcStream(const TPathId& streamPathId) { + auto it = CdcStreams.find(streamPathId); + if (it == CdcStreams.end()) { + return; + } + + it->second.State = TCdcStream::EState::ECdcStreamStateDisabled; + NKikimrSchemeOp::TTableDescription schema; - GetSchema(schema); - - for (auto it = schema.MutableCdcStreams()->begin(); it != schema.MutableCdcStreams()->end(); ++it) { - if (streamPathId != TPathId(it->GetPathId().GetOwnerId(), it->GetPathId().GetLocalId())) { - continue; - } - - it->SetState(TCdcStream::EState::ECdcStreamStateDisabled); - SetSchema(schema); - - return; - } - - Y_FAIL("unreachable"); -} - -void TUserTable::DropCdcStream(const TPathId& streamPathId) { - auto it = CdcStreams.find(streamPathId); - if (it == CdcStreams.end()) { - return; - } - - CdcStreams.erase(it); - + GetSchema(schema); + + for (auto it = schema.MutableCdcStreams()->begin(); it != schema.MutableCdcStreams()->end(); ++it) { + if (streamPathId != TPathId(it->GetPathId().GetOwnerId(), it->GetPathId().GetLocalId())) { + continue; + } + + it->SetState(TCdcStream::EState::ECdcStreamStateDisabled); + SetSchema(schema); + + return; + } + + Y_FAIL("unreachable"); +} + +void TUserTable::DropCdcStream(const TPathId& streamPathId) { + auto it = CdcStreams.find(streamPathId); + if (it == CdcStreams.end()) { + return; + } + + CdcStreams.erase(it); + NKikimrSchemeOp::TTableDescription schema; - GetSchema(schema); - - for (auto it = schema.GetCdcStreams().begin(); it != schema.GetCdcStreams().end(); ++it) { - if (streamPathId != TPathId(it->GetPathId().GetOwnerId(), it->GetPathId().GetLocalId())) { - continue; - } - - schema.MutableCdcStreams()->erase(it); - SetSchema(schema); - - return; - } - - Y_FAIL("unreachable"); -} - -bool TUserTable::HasCdcStreams() const { - return !CdcStreams.empty(); -} - + GetSchema(schema); + + for (auto it = schema.GetCdcStreams().begin(); it != schema.GetCdcStreams().end(); ++it) { + if (streamPathId != TPathId(it->GetPathId().GetOwnerId(), it->GetPathId().GetLocalId())) { + continue; + } + + schema.MutableCdcStreams()->erase(it); + SetSchema(schema); + + return; + } + + Y_FAIL("unreachable"); +} + +bool TUserTable::HasCdcStreams() const { + return !CdcStreams.empty(); +} + void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) { // We expect schemeshard to send us full list of storage rooms @@ -265,17 +265,17 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) IsBackup = descr.GetIsBackup(); CheckSpecialColumns(); - - for (const auto& indexDesc : descr.GetTableIndexes()) { - Y_VERIFY(indexDesc.HasPathOwnerId() && indexDesc.HasLocalPathId()); - Indexes.emplace(TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()), TTableIndex(indexDesc, Columns)); - AsyncIndexCount += ui32(indexDesc.GetType() == TTableIndex::EIndexType::EIndexTypeGlobalAsync); - } - - for (const auto& streamDesc : descr.GetCdcStreams()) { - Y_VERIFY(streamDesc.HasPathId()); - CdcStreams.emplace(TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()), TCdcStream(streamDesc)); - } + + for (const auto& indexDesc : descr.GetTableIndexes()) { + Y_VERIFY(indexDesc.HasPathOwnerId() && indexDesc.HasLocalPathId()); + Indexes.emplace(TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()), TTableIndex(indexDesc, Columns)); + AsyncIndexCount += ui32(indexDesc.GetType() == TTableIndex::EIndexType::EIndexTypeGlobalAsync); + } + + for (const auto& streamDesc : descr.GetCdcStreams()) { + Y_VERIFY(streamDesc.HasPathId()); + CdcStreams.emplace(TPathId(streamDesc.GetPathId().GetOwnerId(), streamDesc.GetPathId().GetLocalId()), TCdcStream(streamDesc)); + } } void TUserTable::CheckSpecialColumns() { diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 46796628a53..0dab30cb84f 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -7,7 +7,7 @@ #include <ydb/core/tablet_flat/flat_stat_table.h> #include <util/generic/ptr.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> namespace NKikimr { @@ -248,58 +248,58 @@ struct TUserTable : public TThrRefBase { {} }; - struct TTableIndex { + struct TTableIndex { using EIndexType = NKikimrSchemeOp::EIndexType; - - TString Name; - EIndexType Type; - TVector<ui32> KeyColumnIds; - TVector<ui32> DataColumnIds; - - TTableIndex() = default; - + + TString Name; + EIndexType Type; + TVector<ui32> KeyColumnIds; + TVector<ui32> DataColumnIds; + + TTableIndex() = default; + TTableIndex(const NKikimrSchemeOp::TIndexDescription& indexDesc, const TMap<ui32, TUserColumn>& columns) - : Name(indexDesc.GetName()) - , Type(indexDesc.GetType()) - { - THashMap<TStringBuf, ui32> nameToId; - for (const auto& [id, column] : columns) { - Y_VERIFY_DEBUG(!nameToId.contains(column.Name)); - nameToId.emplace(column.Name, id); - } - - auto fillColumnIds = [&nameToId](const auto& columnNames, TVector<ui32>& columnIds) { - columnIds.reserve(columnNames.size()); - for (const auto& columnName : columnNames) { - auto it = nameToId.find(columnName); - Y_VERIFY(it != nameToId.end()); - columnIds.push_back(it->second); - } - }; - - fillColumnIds(indexDesc.GetKeyColumnNames(), KeyColumnIds); - fillColumnIds(indexDesc.GetDataColumnNames(), DataColumnIds); + : Name(indexDesc.GetName()) + , Type(indexDesc.GetType()) + { + THashMap<TStringBuf, ui32> nameToId; + for (const auto& [id, column] : columns) { + Y_VERIFY_DEBUG(!nameToId.contains(column.Name)); + nameToId.emplace(column.Name, id); + } + + auto fillColumnIds = [&nameToId](const auto& columnNames, TVector<ui32>& columnIds) { + columnIds.reserve(columnNames.size()); + for (const auto& columnName : columnNames) { + auto it = nameToId.find(columnName); + Y_VERIFY(it != nameToId.end()); + columnIds.push_back(it->second); + } + }; + + fillColumnIds(indexDesc.GetKeyColumnNames(), KeyColumnIds); + fillColumnIds(indexDesc.GetDataColumnNames(), DataColumnIds); } - }; - - struct TCdcStream { + }; + + struct TCdcStream { using EMode = NKikimrSchemeOp::ECdcStreamMode; using EState = NKikimrSchemeOp::ECdcStreamState; - - TString Name; - EMode Mode; - EState State; - - TCdcStream() = default; - + + TString Name; + EMode Mode; + EState State; + + TCdcStream() = default; + TCdcStream(const NKikimrSchemeOp::TCdcStreamDescription& streamDesc) - : Name(streamDesc.GetName()) - , Mode(streamDesc.GetMode()) - , State(streamDesc.GetState()) - { - } - }; - + : Name(streamDesc.GetName()) + , Mode(streamDesc.GetMode()) + , State(streamDesc.GetState()) + { + } + }; + struct TStats { NTable::TStats DataStats; ui64 IndexSize = 0; @@ -353,10 +353,10 @@ struct TUserTable : public TThrRefBase { TVector<ui32> KeyColumnIds; TSerializedTableRange Range; bool IsBackup = false; - - TMap<TPathId, TTableIndex> Indexes; - TMap<TPathId, TCdcStream> CdcStreams; - ui32 AsyncIndexCount = 0; + + TMap<TPathId, TTableIndex> Indexes; + TMap<TPathId, TCdcStream> CdcStreams; + ui32 AsyncIndexCount = 0; // Tablet thread access only, updated in-place mutable TStats Stats; @@ -400,14 +400,14 @@ struct TUserTable : public TThrRefBase { bool ResetTableSchemaVersion(); void AddIndex(const NKikimrSchemeOp::TIndexDescription& indexDesc); - void DropIndex(const TPathId& indexPathId); - bool HasAsyncIndexes() const; - + void DropIndex(const TPathId& indexPathId); + bool HasAsyncIndexes() const; + void AddCdcStream(const NKikimrSchemeOp::TCdcStreamDescription& streamDesc); - void DisableCdcStream(const TPathId& streamPathId); - void DropCdcStream(const TPathId& streamPathId); - bool HasCdcStreams() const; - + void DisableCdcStream(const TPathId& streamPathId); + void DropCdcStream(const TPathId& streamPathId); + bool HasCdcStreams() const; + private: void DoApplyCreate(NTabletFlatExecutor::TTransactionContext& txc, const TString& tableName, bool shadow, const NKikimrSchemeOp::TPartitionConfig& partConfig) const; diff --git a/ydb/core/tx/datashard/datashard_ut_change_collector.cpp b/ydb/core/tx/datashard/datashard_ut_change_collector.cpp index 4a6213dde4a..cdce49ecdb0 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_collector.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_collector.cpp @@ -1,774 +1,774 @@ -#include "change_record.h" -#include "datashard_ut_common.h" - +#include "change_record.h" +#include "datashard_ut_common.h" + #include <ydb/core/protos/change_exchange.pb.h> #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/public/lib/deprecated/kicli/kicli.h> - -namespace NKikimr { - + +namespace NKikimr { + using namespace NDataShard; -using namespace Tests; - -namespace { - -auto GetValueFromLocalDb(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId, const TString& query) { - auto request = MakeHolder<TEvTablet::TEvLocalMKQL>(); - request->Record.MutableProgram()->MutableProgram()->SetText(query); - runtime.SendToPipe(tabletId, sender, request.Release(), 0, GetPipeConfigWithRetries()); - - TAutoPtr<IEventHandle> handle; - auto response = runtime.GrabEdgeEventRethrow<TEvTablet::TEvLocalMKQLResponse>(handle); - UNIT_ASSERT_VALUES_EQUAL(response->Record.GetStatus(), NKikimrProto::OK); - - return response->Record.GetExecutionEngineEvaluatedResponse(); -} - -auto GetChangeRecords(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { - auto protoValue = GetValueFromLocalDb(runtime, sender, tabletId, R"(( - (let range '( '('Order (Uint64 '0) (Void) ))) - (let columns '('Order 'Group 'PlanStep 'TxId 'PathOwnerId 'LocalPathId) ) - (let result (SelectRange 'ChangeRecords range columns '())) - (return (AsList (SetResult 'Result result) )) - ))"); - auto value = NClient::TValue::Create(protoValue); - const auto& result = value["Result"]["List"]; - - TVector<std::tuple<ui64, ui64, ui64, ui64, TPathId>> records; - for (size_t i = 0; i < result.Size(); ++i) { - const auto& item = result[i]; - records.emplace_back( - item["Order"], - item["Group"], - item["PlanStep"], - item["TxId"], - TPathId(item["PathOwnerId"], item["LocalPathId"]) - ); - } - - return records; -} - -auto GetChangeRecordDetails(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { - auto protoValue = GetValueFromLocalDb(runtime, sender, tabletId, R"(( - (let range '( '('Order (Uint64 '0) (Void) ))) - (let columns '('Order 'Kind 'Body) ) - (let result (SelectRange 'ChangeRecordDetails range columns '())) - (return (AsList (SetResult 'Result result) )) - ))"); - auto value = NClient::TValue::Create(protoValue); - const auto& result = value["Result"]["List"]; - - TVector<std::tuple<ui64, TChangeRecord::EKind, TString>> records; - for (size_t i = 0; i < result.Size(); ++i) { - const auto& item = result[i]; - records.emplace_back( - item["Order"], - static_cast<TChangeRecord::EKind>(ui8(item["Kind"])), - item["Body"] - ); - } - - return records; -} - -auto GetChangeRecordsWithDetails(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { - const auto records = GetChangeRecords(runtime, sender, tabletId); - const auto details = GetChangeRecordDetails(runtime, sender, tabletId); - UNIT_ASSERT_VALUES_EQUAL(records.size(), details.size()); - - THashMap<TPathId, TVector<TChangeRecord>> result; - for (size_t i = 0; i < records.size(); ++i) { - const auto& record = records.at(i); - const auto& detail = details.at(i); - UNIT_ASSERT_VALUES_EQUAL(std::get<0>(record), std::get<0>(detail)); - - const auto& pathId = std::get<4>(record); - auto it = result.find(pathId); - if (it == result.end()) { - it = result.emplace(pathId, TVector<TChangeRecord>()).first; - } - - it->second.push_back( - TChangeRecordBuilder(std::get<1>(detail)) - .WithOrder(std::get<0>(record)) - .WithGroup(std::get<1>(record)) - .WithStep(std::get<2>(record)) - .WithTxId(std::get<3>(record)) - .WithBody(std::get<2>(detail)) - .Build() - ); - } - - return result; -} - -THolder<NSchemeCache::TSchemeCacheNavigate> Navigate(TTestActorRuntime& runtime, const TActorId& sender, const TString& path) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; - - auto request = MakeHolder<TNavigate>(); - auto& entry = request->ResultSet.emplace_back(); - entry.Path = SplitPath(path); - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - entry.Operation = TNavigate::EOp::OpTable; - entry.ShowPrivatePath = true; - runtime.Send(new IEventHandle(MakeSchemeCacheID(), sender, new TEvRequest(request.Release()))); - - auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()); - - auto* response = ev->Get()->Request.Release(); - UNIT_ASSERT(response); - UNIT_ASSERT(response->ErrorCount == 0); - UNIT_ASSERT_VALUES_EQUAL(response->ResultSet.size(), 1); - +using namespace Tests; + +namespace { + +auto GetValueFromLocalDb(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId, const TString& query) { + auto request = MakeHolder<TEvTablet::TEvLocalMKQL>(); + request->Record.MutableProgram()->MutableProgram()->SetText(query); + runtime.SendToPipe(tabletId, sender, request.Release(), 0, GetPipeConfigWithRetries()); + + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvTablet::TEvLocalMKQLResponse>(handle); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetStatus(), NKikimrProto::OK); + + return response->Record.GetExecutionEngineEvaluatedResponse(); +} + +auto GetChangeRecords(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { + auto protoValue = GetValueFromLocalDb(runtime, sender, tabletId, R"(( + (let range '( '('Order (Uint64 '0) (Void) ))) + (let columns '('Order 'Group 'PlanStep 'TxId 'PathOwnerId 'LocalPathId) ) + (let result (SelectRange 'ChangeRecords range columns '())) + (return (AsList (SetResult 'Result result) )) + ))"); + auto value = NClient::TValue::Create(protoValue); + const auto& result = value["Result"]["List"]; + + TVector<std::tuple<ui64, ui64, ui64, ui64, TPathId>> records; + for (size_t i = 0; i < result.Size(); ++i) { + const auto& item = result[i]; + records.emplace_back( + item["Order"], + item["Group"], + item["PlanStep"], + item["TxId"], + TPathId(item["PathOwnerId"], item["LocalPathId"]) + ); + } + + return records; +} + +auto GetChangeRecordDetails(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { + auto protoValue = GetValueFromLocalDb(runtime, sender, tabletId, R"(( + (let range '( '('Order (Uint64 '0) (Void) ))) + (let columns '('Order 'Kind 'Body) ) + (let result (SelectRange 'ChangeRecordDetails range columns '())) + (return (AsList (SetResult 'Result result) )) + ))"); + auto value = NClient::TValue::Create(protoValue); + const auto& result = value["Result"]["List"]; + + TVector<std::tuple<ui64, TChangeRecord::EKind, TString>> records; + for (size_t i = 0; i < result.Size(); ++i) { + const auto& item = result[i]; + records.emplace_back( + item["Order"], + static_cast<TChangeRecord::EKind>(ui8(item["Kind"])), + item["Body"] + ); + } + + return records; +} + +auto GetChangeRecordsWithDetails(TTestActorRuntime& runtime, const TActorId& sender, ui64 tabletId) { + const auto records = GetChangeRecords(runtime, sender, tabletId); + const auto details = GetChangeRecordDetails(runtime, sender, tabletId); + UNIT_ASSERT_VALUES_EQUAL(records.size(), details.size()); + + THashMap<TPathId, TVector<TChangeRecord>> result; + for (size_t i = 0; i < records.size(); ++i) { + const auto& record = records.at(i); + const auto& detail = details.at(i); + UNIT_ASSERT_VALUES_EQUAL(std::get<0>(record), std::get<0>(detail)); + + const auto& pathId = std::get<4>(record); + auto it = result.find(pathId); + if (it == result.end()) { + it = result.emplace(pathId, TVector<TChangeRecord>()).first; + } + + it->second.push_back( + TChangeRecordBuilder(std::get<1>(detail)) + .WithOrder(std::get<0>(record)) + .WithGroup(std::get<1>(record)) + .WithStep(std::get<2>(record)) + .WithTxId(std::get<3>(record)) + .WithBody(std::get<2>(detail)) + .Build() + ); + } + + return result; +} + +THolder<NSchemeCache::TSchemeCacheNavigate> Navigate(TTestActorRuntime& runtime, const TActorId& sender, const TString& path) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; + + auto request = MakeHolder<TNavigate>(); + auto& entry = request->ResultSet.emplace_back(); + entry.Path = SplitPath(path); + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + entry.Operation = TNavigate::EOp::OpTable; + entry.ShowPrivatePath = true; + runtime.Send(new IEventHandle(MakeSchemeCacheID(), sender, new TEvRequest(request.Release()))); + + auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()); + + auto* response = ev->Get()->Request.Release(); + UNIT_ASSERT(response); + UNIT_ASSERT(response->ErrorCount == 0); + UNIT_ASSERT_VALUES_EQUAL(response->ResultSet.size(), 1); + return THolder(response); -} - -using TStructKey = TVector<std::pair<TString, ui32>>; -using TStructValue = THashMap<TString, ui32>; -constexpr ui32 Null = 0; - -template <typename C> -static void OutKvContainer(IOutputStream& out, const C& c) { - out << "{"; - for (const auto& [k, v] : c) { - out << " (" << k << ": "; - if (v == Null) { - out << "null"; - } else { - out << v; - } - out << ")"; - } - out << " }"; -} - -struct TStructRecord { +} + +using TStructKey = TVector<std::pair<TString, ui32>>; +using TStructValue = THashMap<TString, ui32>; +constexpr ui32 Null = 0; + +template <typename C> +static void OutKvContainer(IOutputStream& out, const C& c) { + out << "{"; + for (const auto& [k, v] : c) { + out << " (" << k << ": "; + if (v == Null) { + out << "null"; + } else { + out << v; + } + out << ")"; + } + out << " }"; +} + +struct TStructRecord { NTable::ERowOp Rop; - TStructKey Key; - TStructValue Update; - TStructValue OldImage; - TStructValue NewImage; - - TStructRecord() = default; - + TStructKey Key; + TStructValue Update; + TStructValue OldImage; + TStructValue NewImage; + + TStructRecord() = default; + TStructRecord(NTable::ERowOp rop, const TStructKey& key, - const TStructValue& update = {}, - const TStructValue& oldImage = {}, - const TStructValue& newImage = {}) - : Rop(rop) - , Key(key) - , Update(update) - , OldImage(oldImage) - , NewImage(newImage) - { - } - - bool operator==(const TStructRecord& rhs) const { - return Rop == rhs.Rop - && Key == rhs.Key - && Update == rhs.Update - && OldImage == rhs.OldImage - && NewImage == rhs.NewImage; - } - - void Out(IOutputStream& out) const { - out << "{" - << " Rop: " << Rop - << " Key: " << Key - << " Update: " << Update - << " OldImage: " << OldImage - << " NewImage: " << NewImage - << " }"; - } - - static TStructRecord Parse(const NKikimrChangeExchange::TChangeRecord::TDataChange& proto, - const THashMap<NTable::TTag, TString>& tagToName) - { - TStructRecord record; - - Parse(proto.GetKey(), tagToName, [&record](const TString& name, ui32 value) { - record.Key.emplace_back(name, value); - }); - - switch (proto.GetRowOperationCase()) { - case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + const TStructValue& update = {}, + const TStructValue& oldImage = {}, + const TStructValue& newImage = {}) + : Rop(rop) + , Key(key) + , Update(update) + , OldImage(oldImage) + , NewImage(newImage) + { + } + + bool operator==(const TStructRecord& rhs) const { + return Rop == rhs.Rop + && Key == rhs.Key + && Update == rhs.Update + && OldImage == rhs.OldImage + && NewImage == rhs.NewImage; + } + + void Out(IOutputStream& out) const { + out << "{" + << " Rop: " << Rop + << " Key: " << Key + << " Update: " << Update + << " OldImage: " << OldImage + << " NewImage: " << NewImage + << " }"; + } + + static TStructRecord Parse(const NKikimrChangeExchange::TChangeRecord::TDataChange& proto, + const THashMap<NTable::TTag, TString>& tagToName) + { + TStructRecord record; + + Parse(proto.GetKey(), tagToName, [&record](const TString& name, ui32 value) { + record.Key.emplace_back(name, value); + }); + + switch (proto.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: record.Rop = NTable::ERowOp::Upsert; - Parse(proto.GetUpsert(), tagToName, [&record](const TString& name, ui32 value) { - record.Update.emplace(name, value); - }); - break; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: + Parse(proto.GetUpsert(), tagToName, [&record](const TString& name, ui32 value) { + record.Update.emplace(name, value); + }); + break; + case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: record.Rop = NTable::ERowOp::Erase; - break; - default: + break; + default: record.Rop = NTable::ERowOp::Absent; - break; - } - - if (proto.HasOldImage()) { - Parse(proto.GetOldImage(), tagToName, [&record](const TString& name, ui32 value) { - record.OldImage.emplace(name, value); - }); - } - - if (proto.HasNewImage()) { - Parse(proto.GetNewImage(), tagToName, [&record](const TString& name, ui32 value) { - record.NewImage.emplace(name, value); - }); - } - - return record; - } - - static TStructRecord Parse(const TString& serializedProto, const THashMap<NTable::TTag, TString>& tagToName) { - NKikimrChangeExchange::TChangeRecord::TDataChange proto; + break; + } + + if (proto.HasOldImage()) { + Parse(proto.GetOldImage(), tagToName, [&record](const TString& name, ui32 value) { + record.OldImage.emplace(name, value); + }); + } + + if (proto.HasNewImage()) { + Parse(proto.GetNewImage(), tagToName, [&record](const TString& name, ui32 value) { + record.NewImage.emplace(name, value); + }); + } + + return record; + } + + static TStructRecord Parse(const TString& serializedProto, const THashMap<NTable::TTag, TString>& tagToName) { + NKikimrChangeExchange::TChangeRecord::TDataChange proto; Y_PROTOBUF_SUPPRESS_NODISCARD proto.ParseFromArray(serializedProto.data(), serializedProto.size()); - return Parse(proto, tagToName); - } - -private: - using TInserter = std::function<void(const TString&, ui32)>; - - static void Parse(const NKikimrChangeExchange::TChangeRecord::TDataChange::TSerializedCells& proto, - const THashMap<NTable::TTag, TString>& tagToName, TInserter inserter) - { - TSerializedCellVec serialized; - UNIT_ASSERT(TSerializedCellVec::TryParse(proto.GetData(), serialized)); - - const auto& cells = serialized.GetCells(); - UNIT_ASSERT_VALUES_EQUAL(cells.size(), proto.TagsSize()); - - for (ui32 i = 0; i < proto.TagsSize(); ++i) { - const auto tag = proto.GetTags(i); - if (!tagToName.contains(tag)) { - continue; - } - - const auto& name = tagToName.at(tag); - const auto& cell = cells.at(i); - - if (cell.IsNull()) { - inserter(name, Null); - } else { - inserter(name, cell.AsValue<ui32>()); - } - } - } -}; - -using TStructRecords = THashMap<TString, TVector<TStructRecord>>; - -} // anonymous - -Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { - void Run(const TString& path, const TShardedTableOptions& opts, const TVector<TString>& queries, const TStructRecords& expectedRecords) { - const auto pathParts = SplitPath(path); - UNIT_ASSERT(pathParts.size() > 1); - - const auto domainName = pathParts.at(0); - const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); - const auto tableName = pathParts.at(pathParts.size() - 1); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName(domainName) - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - // prevent change sending - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::TEvActivateSender::EventType: - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - CreateShardedTable(server, sender, workingDir, tableName, opts); - for (const auto& query : queries) { - ExecSQL(server, sender, query); - } - - auto desc = Navigate(runtime, sender, path); - const auto& entry = desc->ResultSet.at(0); - - THashMap<NTable::TTag, TString> tagToName; - for (const auto& [tag, column] : entry.Columns) { - tagToName.emplace(tag, column.Name); - } - - THashMap<TPathId, TString> indexPathIdToName; - for (const auto& index : entry.Indexes) { - const auto& name = index.GetName(); - const auto pathId = TPathId(index.GetPathOwnerId(), index.GetLocalPathId()); - indexPathIdToName.emplace(pathId, name); - } - - const auto tabletIds = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - - for (const auto& [pathId, actual] : GetChangeRecordsWithDetails(runtime, sender, tabletIds[0])) { - UNIT_ASSERT(indexPathIdToName.contains(pathId)); - const auto& name = indexPathIdToName.at(pathId); - - UNIT_ASSERT(expectedRecords.contains(name)); - const auto& expected = expectedRecords.at(name); - - UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); - for (size_t i = 0; i < expected.size(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(expected.at(i), TStructRecord::Parse(actual.at(i).GetBody(), tagToName)); - } - } - } - - void Run(const TString& path, const TShardedTableOptions& opts, const TString& query, const TStructRecords& expectedRecords) { - Run(path, opts, TVector<TString>(1, query), expectedRecords); - } - - TShardedTableOptions SimpleTable() { - return TShardedTableOptions() - .Columns({ + return Parse(proto, tagToName); + } + +private: + using TInserter = std::function<void(const TString&, ui32)>; + + static void Parse(const NKikimrChangeExchange::TChangeRecord::TDataChange::TSerializedCells& proto, + const THashMap<NTable::TTag, TString>& tagToName, TInserter inserter) + { + TSerializedCellVec serialized; + UNIT_ASSERT(TSerializedCellVec::TryParse(proto.GetData(), serialized)); + + const auto& cells = serialized.GetCells(); + UNIT_ASSERT_VALUES_EQUAL(cells.size(), proto.TagsSize()); + + for (ui32 i = 0; i < proto.TagsSize(); ++i) { + const auto tag = proto.GetTags(i); + if (!tagToName.contains(tag)) { + continue; + } + + const auto& name = tagToName.at(tag); + const auto& cell = cells.at(i); + + if (cell.IsNull()) { + inserter(name, Null); + } else { + inserter(name, cell.AsValue<ui32>()); + } + } + } +}; + +using TStructRecords = THashMap<TString, TVector<TStructRecord>>; + +} // anonymous + +Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { + void Run(const TString& path, const TShardedTableOptions& opts, const TVector<TString>& queries, const TStructRecords& expectedRecords) { + const auto pathParts = SplitPath(path); + UNIT_ASSERT(pathParts.size() > 1); + + const auto domainName = pathParts.at(0); + const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); + const auto tableName = pathParts.at(pathParts.size() - 1); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName(domainName) + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + // prevent change sending + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::TEvActivateSender::EventType: + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + CreateShardedTable(server, sender, workingDir, tableName, opts); + for (const auto& query : queries) { + ExecSQL(server, sender, query); + } + + auto desc = Navigate(runtime, sender, path); + const auto& entry = desc->ResultSet.at(0); + + THashMap<NTable::TTag, TString> tagToName; + for (const auto& [tag, column] : entry.Columns) { + tagToName.emplace(tag, column.Name); + } + + THashMap<TPathId, TString> indexPathIdToName; + for (const auto& index : entry.Indexes) { + const auto& name = index.GetName(); + const auto pathId = TPathId(index.GetPathOwnerId(), index.GetLocalPathId()); + indexPathIdToName.emplace(pathId, name); + } + + const auto tabletIds = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + + for (const auto& [pathId, actual] : GetChangeRecordsWithDetails(runtime, sender, tabletIds[0])) { + UNIT_ASSERT(indexPathIdToName.contains(pathId)); + const auto& name = indexPathIdToName.at(pathId); + + UNIT_ASSERT(expectedRecords.contains(name)); + const auto& expected = expectedRecords.at(name); + + UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); + for (size_t i = 0; i < expected.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(expected.at(i), TStructRecord::Parse(actual.at(i).GetBody(), tagToName)); + } + } + } + + void Run(const TString& path, const TShardedTableOptions& opts, const TString& query, const TStructRecords& expectedRecords) { + Run(path, opts, TVector<TString>(1, query), expectedRecords); + } + + TShardedTableOptions SimpleTable() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }); - } - - Y_UNIT_TEST(InsertSingleRow) { - Run("/Root/path", SimpleTable(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { + }); + } + + Y_UNIT_TEST(InsertSingleRow) { + Run("/Root/path", SimpleTable(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { {"by_ikey", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}})}}, - }); - } - - Y_UNIT_TEST(InsertManyRows) { - Run("/Root/path", SimpleTable(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (2, 20);", { - {"by_ikey", { + }); + } + + Y_UNIT_TEST(InsertManyRows) { + Run("/Root/path", SimpleTable(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (2, 20);", { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 20}, {"pkey", 2}}), - }}, - }); - } - - Y_UNIT_TEST(UpsertSingleRow) { - Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { + }}, + }); + } + + Y_UNIT_TEST(UpsertSingleRow) { + Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { {"by_ikey", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}})}}, - }); - } - - Y_UNIT_TEST(UpsertManyRows) { - Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (2, 20);", { - {"by_ikey", { + }); + } + + Y_UNIT_TEST(UpsertManyRows) { + Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (2, 20);", { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 20}, {"pkey", 2}}), - }}, - }); - } - - Y_UNIT_TEST(UpsertToSameKey) { - Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (1, 20);", { - {"by_ikey", { + }}, + }); + } + + Y_UNIT_TEST(UpsertToSameKey) { + Run("/Root/path", SimpleTable(), "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10), (1, 20);", { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 20}, {"pkey", 1}}), - }}, - }); - } - - Y_UNIT_TEST(DeleteNothing) { - Run("/Root/path", SimpleTable(), "DELETE FROM `/Root/path` WHERE pkey = 1;", {}); - } - - Y_UNIT_TEST(DeleteSingleRow) { - Run("/Root/path", SimpleTable(), TVector<TString>{ - "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", - "DELETE FROM `/Root/path` WHERE pkey = 1;", - }, { - {"by_ikey", { + }}, + }); + } + + Y_UNIT_TEST(DeleteNothing) { + Run("/Root/path", SimpleTable(), "DELETE FROM `/Root/path` WHERE pkey = 1;", {}); + } + + Y_UNIT_TEST(DeleteSingleRow) { + Run("/Root/path", SimpleTable(), TVector<TString>{ + "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", + "DELETE FROM `/Root/path` WHERE pkey = 1;", + }, { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey", 10}, {"pkey", 1}}), - }}, - }); - } - - TShardedTableOptions MultiIndexedTable() { - return TShardedTableOptions() - .Columns({ + }}, + }); + } + + TShardedTableOptions MultiIndexedTable() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey1", "Uint32", false, false}, {"ikey2", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_ikey1", {"ikey1"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, {"by_ikey2", {"ikey2"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }); - } - - Y_UNIT_TEST(MultiIndexedTableInsertSingleRow) { - Run("/Root/path", MultiIndexedTable(), "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", { + }); + } + + Y_UNIT_TEST(MultiIndexedTableInsertSingleRow) { + Run("/Root/path", MultiIndexedTable(), "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", { {"by_ikey1", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey1", 10}, {"pkey", 1}})}}, {"by_ikey2", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey2", 100}, {"pkey", 1}})}}, - }); - } - - Y_UNIT_TEST(MultiIndexedTableUpdateOneIndexedColumn) { - Run("/Root/path", MultiIndexedTable(), TVector<TString>{ - "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", - "UPDATE `/Root/path` SET ikey1 = 20 WHERE pkey = 1;", - }, { - {"by_ikey1", { + }); + } + + Y_UNIT_TEST(MultiIndexedTableUpdateOneIndexedColumn) { + Run("/Root/path", MultiIndexedTable(), TVector<TString>{ + "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", + "UPDATE `/Root/path` SET ikey1 = 20 WHERE pkey = 1;", + }, { + {"by_ikey1", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey1", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey1", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey1", 20}, {"pkey", 1}}), - }}, - {"by_ikey2", { + }}, + {"by_ikey2", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey2", 100}, {"pkey", 1}}), - }}, - }); - } - - Y_UNIT_TEST(MultiIndexedTableReplaceSingleRow) { - Run("/Root/path", MultiIndexedTable(), TVector<TString>{ - "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", - "REPLACE INTO `/Root/path` (pkey, ikey1) VALUES (1, 20);", - }, { - {"by_ikey1", { + }}, + }); + } + + Y_UNIT_TEST(MultiIndexedTableReplaceSingleRow) { + Run("/Root/path", MultiIndexedTable(), TVector<TString>{ + "INSERT INTO `/Root/path` (pkey, ikey1, ikey2) VALUES (1, 10, 100);", + "REPLACE INTO `/Root/path` (pkey, ikey1) VALUES (1, 20);", + }, { + {"by_ikey1", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey1", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey1", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey1", 20}, {"pkey", 1}}), - }}, - {"by_ikey2", { + }}, + {"by_ikey2", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey2", 100}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey2", 100}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey2", Null}, {"pkey", 1}}), - }}, - }); - } - - TShardedTableOptions IndexedPrimaryKey() { - return TShardedTableOptions() - .Columns({ + }}, + }); + } + + TShardedTableOptions IndexedPrimaryKey() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_ikey_pkey", {"ikey", "pkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }); - } - - Y_UNIT_TEST(IndexedPrimaryKeyInsertSingleRow) { - Run("/Root/path", IndexedPrimaryKey(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { + }); + } + + Y_UNIT_TEST(IndexedPrimaryKeyInsertSingleRow) { + Run("/Root/path", IndexedPrimaryKey(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { {"by_ikey_pkey", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}})}}, - }); - } - - Y_UNIT_TEST(IndexedPrimaryKeyDeleteSingleRow) { - Run("/Root/path", IndexedPrimaryKey(), TVector<TString>{ - "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", - "DELETE FROM `/Root/path` WHERE pkey = 1;", - }, { - {"by_ikey_pkey", { + }); + } + + Y_UNIT_TEST(IndexedPrimaryKeyDeleteSingleRow) { + Run("/Root/path", IndexedPrimaryKey(), TVector<TString>{ + "UPSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", + "DELETE FROM `/Root/path` WHERE pkey = 1;", + }, { + {"by_ikey_pkey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey", 10}, {"pkey", 1}}), - }}, - }); - } - - TShardedTableOptions CoveredIndex() { - return TShardedTableOptions() - .Columns({ + }}, + }); + } + + TShardedTableOptions CoveredIndex() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, {"value", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_ikey", {"ikey"}, {"value"}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }); - } - - Y_UNIT_TEST(CoveredIndexUpdateCoveredColumn) { - Run("/Root/path", CoveredIndex(), TVector<TString>{ - "INSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 100);", - "UPDATE `/Root/path` SET value = 200 WHERE pkey = 1;", - }, { - {"by_ikey", { + }); + } + + Y_UNIT_TEST(CoveredIndexUpdateCoveredColumn) { + Run("/Root/path", CoveredIndex(), TVector<TString>{ + "INSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 100);", + "UPDATE `/Root/path` SET value = 200 WHERE pkey = 1;", + }, { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}, {{"value", 100}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}, {{"value", 200}}), - }}, - }); - } - - Y_UNIT_TEST(CoveredIndexUpsert) { - Run("/Root/path", CoveredIndex(), TVector<TString>{ - "INSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 100);", - "UPSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 200);", - }, { - {"by_ikey", { + }}, + }); + } + + Y_UNIT_TEST(CoveredIndexUpsert) { + Run("/Root/path", CoveredIndex(), TVector<TString>{ + "INSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 100);", + "UPSERT INTO `/Root/path` (pkey, ikey, value) VALUES (1, 10, 200);", + }, { + {"by_ikey", { TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}, {{"value", 100}}), TStructRecord(NTable::ERowOp::Erase, {{"ikey", 10}, {"pkey", 1}}), TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}}, {{"value", 200}}), - }}, - }); - } -} - -Y_UNIT_TEST_SUITE(CdcStreamChangeCollector) { - using TCdcStream = TShardedTableOptions::TCdcStream; - - NKikimrPQ::TPQConfig WithProtoSourceIdInfo() { - NKikimrPQ::TPQConfig pqConfig; - pqConfig.SetEnableProtoSourceIdInfo(true); - return pqConfig; - } - - void Run(const TString& path, const TShardedTableOptions& opts, const TVector<TCdcStream>& streams, - const TVector<TString>& queries, const TStructRecords& expectedRecords) - { - const auto pathParts = SplitPath(path); - UNIT_ASSERT(pathParts.size() > 1); - - const auto domainName = pathParts.at(0); - const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); - const auto tableName = pathParts.at(pathParts.size() - 1); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134), {}, WithProtoSourceIdInfo()); - serverSettings - .SetDomainName(domainName) - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - // prevent change sending - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::TEvActivateSender::EventType: - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - CreateShardedTable(server, sender, workingDir, tableName, opts); - - for (const auto& stream : streams) { - WaitTxNotification(server, sender, AsyncAlterAddStream(server, workingDir, tableName, stream)); - } - - for (const auto& query : queries) { - ExecSQL(server, sender, query); - } - - auto desc = Navigate(runtime, sender, path); - const auto& entry = desc->ResultSet.at(0); - - THashMap<NTable::TTag, TString> tagToName; - for (const auto& [tag, column] : entry.Columns) { - tagToName.emplace(tag, column.Name); - } - - THashMap<TPathId, TString> indexPathIdToName; - for (const auto& index : entry.Indexes) { - const auto& name = index.GetName(); - const auto pathId = TPathId(index.GetPathOwnerId(), index.GetLocalPathId()); - indexPathIdToName.emplace(pathId, name); - } - - THashMap<TPathId, TString> streamPathIdToName; - for (const auto& stream : entry.CdcStreams) { - const auto& name = stream.GetName(); - const auto pathId = TPathId(stream.GetPathId().GetOwnerId(), stream.GetPathId().GetLocalId()); - streamPathIdToName.emplace(pathId, name); - } - - const auto tabletIds = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - - for (const auto& [pathId, actual] : GetChangeRecordsWithDetails(runtime, sender, tabletIds[0])) { - TString name; - if (streamPathIdToName.contains(pathId)) { - name = streamPathIdToName.at(pathId); - } else if (indexPathIdToName.contains(pathId)) { - name = indexPathIdToName.at(pathId); - } else { - UNIT_ASSERT_C(false, "Unexpected path id: " << pathId); - } - - UNIT_ASSERT(expectedRecords.contains(name)); - const auto& expected = expectedRecords.at(name); - - UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); - for (size_t i = 0; i < expected.size(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(expected.at(i), TStructRecord::Parse(actual.at(i).GetBody(), tagToName)); - } - } - } - - void Run(const TString& path, const TShardedTableOptions& opts, const TCdcStream& stream, - const TString& query, const TStructRecords& expectedRecords) - { - Run(path, opts, TVector<TCdcStream>(1, stream), TVector<TString>(1, query), expectedRecords); - } - - TShardedTableOptions SimpleTable() { - return TShardedTableOptions() - .Columns({ + }}, + }); + } +} + +Y_UNIT_TEST_SUITE(CdcStreamChangeCollector) { + using TCdcStream = TShardedTableOptions::TCdcStream; + + NKikimrPQ::TPQConfig WithProtoSourceIdInfo() { + NKikimrPQ::TPQConfig pqConfig; + pqConfig.SetEnableProtoSourceIdInfo(true); + return pqConfig; + } + + void Run(const TString& path, const TShardedTableOptions& opts, const TVector<TCdcStream>& streams, + const TVector<TString>& queries, const TStructRecords& expectedRecords) + { + const auto pathParts = SplitPath(path); + UNIT_ASSERT(pathParts.size() > 1); + + const auto domainName = pathParts.at(0); + const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); + const auto tableName = pathParts.at(pathParts.size() - 1); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134), {}, WithProtoSourceIdInfo()); + serverSettings + .SetDomainName(domainName) + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + // prevent change sending + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::TEvActivateSender::EventType: + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + CreateShardedTable(server, sender, workingDir, tableName, opts); + + for (const auto& stream : streams) { + WaitTxNotification(server, sender, AsyncAlterAddStream(server, workingDir, tableName, stream)); + } + + for (const auto& query : queries) { + ExecSQL(server, sender, query); + } + + auto desc = Navigate(runtime, sender, path); + const auto& entry = desc->ResultSet.at(0); + + THashMap<NTable::TTag, TString> tagToName; + for (const auto& [tag, column] : entry.Columns) { + tagToName.emplace(tag, column.Name); + } + + THashMap<TPathId, TString> indexPathIdToName; + for (const auto& index : entry.Indexes) { + const auto& name = index.GetName(); + const auto pathId = TPathId(index.GetPathOwnerId(), index.GetLocalPathId()); + indexPathIdToName.emplace(pathId, name); + } + + THashMap<TPathId, TString> streamPathIdToName; + for (const auto& stream : entry.CdcStreams) { + const auto& name = stream.GetName(); + const auto pathId = TPathId(stream.GetPathId().GetOwnerId(), stream.GetPathId().GetLocalId()); + streamPathIdToName.emplace(pathId, name); + } + + const auto tabletIds = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + + for (const auto& [pathId, actual] : GetChangeRecordsWithDetails(runtime, sender, tabletIds[0])) { + TString name; + if (streamPathIdToName.contains(pathId)) { + name = streamPathIdToName.at(pathId); + } else if (indexPathIdToName.contains(pathId)) { + name = indexPathIdToName.at(pathId); + } else { + UNIT_ASSERT_C(false, "Unexpected path id: " << pathId); + } + + UNIT_ASSERT(expectedRecords.contains(name)); + const auto& expected = expectedRecords.at(name); + + UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); + for (size_t i = 0; i < expected.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(expected.at(i), TStructRecord::Parse(actual.at(i).GetBody(), tagToName)); + } + } + } + + void Run(const TString& path, const TShardedTableOptions& opts, const TCdcStream& stream, + const TString& query, const TStructRecords& expectedRecords) + { + Run(path, opts, TVector<TCdcStream>(1, stream), TVector<TString>(1, query), expectedRecords); + } + + TShardedTableOptions SimpleTable() { + return TShardedTableOptions() + .Columns({ {"key", "Uint32", true, false}, {"value", "Uint32", false, false}, - }); - } - - TCdcStream KeysOnly() { - return TCdcStream{ + }); + } + + TCdcStream KeysOnly() { + return TCdcStream{ "keys_stream", NKikimrSchemeOp::ECdcStreamModeKeysOnly - }; - } - - TCdcStream Updates() { - return TCdcStream{ + }; + } + + TCdcStream Updates() { + return TCdcStream{ "updates_stream", NKikimrSchemeOp::ECdcStreamModeUpdate - }; - } - - TCdcStream NewAndOldImages() { - return TCdcStream{ + }; + } + + TCdcStream NewAndOldImages() { + return TCdcStream{ "new_and_old_images", NKikimrSchemeOp::ECdcStreamModeNewAndOldImages - }; - } - - Y_UNIT_TEST(InsertSingleRow) { - Run("/Root/path", SimpleTable(), KeysOnly(), "INSERT INTO `/Root/path` (key, value) VALUES (1, 10);", { + }; + } + + Y_UNIT_TEST(InsertSingleRow) { + Run("/Root/path", SimpleTable(), KeysOnly(), "INSERT INTO `/Root/path` (key, value) VALUES (1, 10);", { {"keys_stream", {TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}})}}, - }); - } - - Y_UNIT_TEST(DeleteNothing) { - Run("/Root/path", SimpleTable(), KeysOnly(), "DELETE FROM `/Root/path` WHERE key = 1;", { + }); + } + + Y_UNIT_TEST(DeleteNothing) { + Run("/Root/path", SimpleTable(), KeysOnly(), "DELETE FROM `/Root/path` WHERE key = 1;", { {"keys_stream", {TStructRecord(NTable::ERowOp::Erase, {{"key", 1}})}}, - }); - } - - Y_UNIT_TEST(UpsertManyRows) { - Run("/Root/path", SimpleTable(), Updates(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (2, 20);", { - {"updates_stream", { + }); + } + + Y_UNIT_TEST(UpsertManyRows) { + Run("/Root/path", SimpleTable(), Updates(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (2, 20);", { + {"updates_stream", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 2}}, {{"value", 20}}), - }}, - }); - } - - Y_UNIT_TEST(UpsertToSameKey) { - Run("/Root/path", SimpleTable(), Updates(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", { - {"updates_stream", { + }}, + }); + } + + Y_UNIT_TEST(UpsertToSameKey) { + Run("/Root/path", SimpleTable(), Updates(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", { + {"updates_stream", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {{"value", 20}}), - }}, - }); - } - - Y_UNIT_TEST(UpsertToSameKeyWithImages) { - Run("/Root/path", SimpleTable(), NewAndOldImages(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", { - {"new_and_old_images", { + }}, + }); + } + + Y_UNIT_TEST(UpsertToSameKeyWithImages) { + Run("/Root/path", SimpleTable(), NewAndOldImages(), "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", { + {"new_and_old_images", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {{"value", 10}}, {{"value", 20}}), - }}, - }); - } - - Y_UNIT_TEST(UpsertIntoTwoStreams) { - Run("/Root/path", SimpleTable(), TVector<TCdcStream>{Updates(), NewAndOldImages()}, TVector<TString>{ - "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", - }, { - {"updates_stream", { + }}, + }); + } + + Y_UNIT_TEST(UpsertIntoTwoStreams) { + Run("/Root/path", SimpleTable(), TVector<TCdcStream>{Updates(), NewAndOldImages()}, TVector<TString>{ + "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10), (1, 20);", + }, { + {"updates_stream", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {{"value", 20}}), - }}, - {"new_and_old_images", { + }}, + {"new_and_old_images", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {{"value", 10}}, {{"value", 20}}), - }}, - }); - } - - Y_UNIT_TEST(DeleteSingleRow) { - Run("/Root/path", SimpleTable(), TVector<TCdcStream>{NewAndOldImages()}, TVector<TString>{ - "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10);", - "DELETE FROM `/Root/path` WHERE key = 1;", - }, { - {"new_and_old_images", { + }}, + }); + } + + Y_UNIT_TEST(DeleteSingleRow) { + Run("/Root/path", SimpleTable(), TVector<TCdcStream>{NewAndOldImages()}, TVector<TString>{ + "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10);", + "DELETE FROM `/Root/path` WHERE key = 1;", + }, { + {"new_and_old_images", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Erase, {{"key", 1}}, {}, {{"value", 10}}, {}), - }}, - }); - } - - Y_UNIT_TEST(UpsertModifyDelete) { - Run("/Root/path", SimpleTable(), TVector<TCdcStream>{NewAndOldImages()}, TVector<TString>{ - "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10);", - "UPSERT INTO `/Root/path` (key, value) VALUES (1, 20);", - "DELETE FROM `/Root/path` WHERE key = 1;", - }, { - {"new_and_old_images", { + }}, + }); + } + + Y_UNIT_TEST(UpsertModifyDelete) { + Run("/Root/path", SimpleTable(), TVector<TCdcStream>{NewAndOldImages()}, TVector<TString>{ + "UPSERT INTO `/Root/path` (key, value) VALUES (1, 10);", + "UPSERT INTO `/Root/path` (key, value) VALUES (1, 20);", + "DELETE FROM `/Root/path` WHERE key = 1;", + }, { + {"new_and_old_images", { TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {}, {{"value", 10}}), TStructRecord(NTable::ERowOp::Upsert, {{"key", 1}}, {}, {{"value", 10}}, {{"value", 20}}), TStructRecord(NTable::ERowOp::Erase, {{"key", 1}}, {}, {{"value", 20}}, {}), - }}, - }); - } - - TShardedTableOptions IndexedTable() { - return TShardedTableOptions() - .Columns({ + }}, + }); + } + + TShardedTableOptions IndexedTable() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }); - } - - Y_UNIT_TEST(IndexAndStreamUpsert) { - Run("/Root/path", IndexedTable(), Updates(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { + }); + } + + Y_UNIT_TEST(IndexAndStreamUpsert) { + Run("/Root/path", IndexedTable(), Updates(), "INSERT INTO `/Root/path` (pkey, ikey) VALUES (1, 10);", { {"by_ikey", {TStructRecord(NTable::ERowOp::Upsert, {{"ikey", 10}, {"pkey", 1}})}}, {"updates_stream", {TStructRecord(NTable::ERowOp::Upsert, {{"pkey", 1}}, {{"ikey", 10}})}}, - }); - } -} - -} // NKikimr - -Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructRecord, out, value) { - return value.Out(out); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructKey, out, value) { - return NKikimr::OutKvContainer(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructValue, out, value) { - return NKikimr::OutKvContainer(out, value); -} + }); + } +} + +} // NKikimr + +Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructRecord, out, value) { + return value.Out(out); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructKey, out, value) { + return NKikimr::OutKvContainer(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimr::TStructValue, out, value) { + return NKikimr::OutKvContainer(out, value); +} diff --git a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp index f5479dad020..5e312f673c4 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp @@ -1,561 +1,561 @@ -#include "datashard_ut_common.h" - -#include <util/string/strip.h> - -namespace NKikimr { - +#include "datashard_ut_common.h" + +#include <util/string/strip.h> + +namespace NKikimr { + using namespace NDataShard; -using namespace Tests; - -Y_UNIT_TEST_SUITE(AsyncIndexChangeExchange) { - void SenderShouldBeActivated(const TString& path, const TShardedTableOptions& opts) { - const auto pathParts = SplitPath(path); - UNIT_ASSERT(pathParts.size() > 1); - - const auto domainName = pathParts.at(0); - const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); - const auto tableName = pathParts.at(pathParts.size() - 1); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName(domainName) - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - bool activated = false; - runtime.SetObserverFunc([&activated](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvChangeExchange::EvActivateSender) { - activated = true; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - CreateShardedTable(server, sender, workingDir, tableName, opts); - - if (!activated) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&activated](IEventHandle&) { - return activated; - }); - server->GetRuntime()->DispatchEvents(opts); - } - } - - TShardedTableOptions TableWoIndexes() { - return TShardedTableOptions() - .Columns({ +using namespace Tests; + +Y_UNIT_TEST_SUITE(AsyncIndexChangeExchange) { + void SenderShouldBeActivated(const TString& path, const TShardedTableOptions& opts) { + const auto pathParts = SplitPath(path); + UNIT_ASSERT(pathParts.size() > 1); + + const auto domainName = pathParts.at(0); + const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); + const auto tableName = pathParts.at(pathParts.size() - 1); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName(domainName) + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + bool activated = false; + runtime.SetObserverFunc([&activated](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvChangeExchange::EvActivateSender) { + activated = true; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + CreateShardedTable(server, sender, workingDir, tableName, opts); + + if (!activated) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&activated](IEventHandle&) { + return activated; + }); + server->GetRuntime()->DispatchEvents(opts); + } + } + + TShardedTableOptions TableWoIndexes() { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, - }); - } - - TShardedTableOptions::TIndex SimpleSyncIndex() { - return TShardedTableOptions::TIndex{ + }); + } + + TShardedTableOptions::TIndex SimpleSyncIndex() { + return TShardedTableOptions::TIndex{ "by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobal - }; - } - - TShardedTableOptions::TIndex SimpleAsyncIndex() { - return TShardedTableOptions::TIndex{ + }; + } + + TShardedTableOptions::TIndex SimpleAsyncIndex() { + return TShardedTableOptions::TIndex{ "by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync - }; - } - - TShardedTableOptions TableWithIndex(const TShardedTableOptions::TIndex& index) { - return TShardedTableOptions() - .Columns({ + }; + } + + TShardedTableOptions TableWithIndex(const TShardedTableOptions::TIndex& index) { + return TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"ikey", "Uint32", false, false}, - }) - .Indexes({ - index - }); - } - - Y_UNIT_TEST(SenderShouldBeActivatedOnTableWoIndexes) { - SenderShouldBeActivated("/Root/Table", TableWoIndexes()); - } - - Y_UNIT_TEST(SenderShouldBeActivatedOnTableWithSyncIndex) { - SenderShouldBeActivated("/Root/Table", TableWithIndex(SimpleSyncIndex())); - } - - Y_UNIT_TEST(SenderShouldBeActivatedOnTableWithAsyncIndex) { - SenderShouldBeActivated("/Root/Table", TableWithIndex(SimpleAsyncIndex())); - } - - void SenderShouldShakeHands(const TString& path, ui32 times, const TShardedTableOptions& opts, - TMaybe<TShardedTableOptions::TIndex> addIndex = Nothing()) - { - const auto pathParts = SplitPath(path); - UNIT_ASSERT(pathParts.size() > 1); - - const auto domainName = pathParts.at(0); - const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); - const auto tableName = pathParts.at(pathParts.size() - 1); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName(domainName) - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - ui32 counter = 0; - runtime.SetObserverFunc([&counter](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvChangeExchange::EvHandshake) { - ++counter; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - CreateShardedTable(server, sender, workingDir, tableName, opts); - if (addIndex) { - AsyncAlterAddIndex(server, domainName, path, *addIndex); - } - - if (counter != times) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&](IEventHandle&) { - return counter == times; - }); - server->GetRuntime()->DispatchEvents(opts); - } - } - - Y_UNIT_TEST(SenderShouldShakeHandsOnce) { - SenderShouldShakeHands("/Root/Table", 1, TableWithIndex(SimpleAsyncIndex())); - } - - Y_UNIT_TEST(SenderShouldShakeHandsTwice) { - SenderShouldShakeHands("/Root/Table", 2, TShardedTableOptions() - .Columns({ + }) + .Indexes({ + index + }); + } + + Y_UNIT_TEST(SenderShouldBeActivatedOnTableWoIndexes) { + SenderShouldBeActivated("/Root/Table", TableWoIndexes()); + } + + Y_UNIT_TEST(SenderShouldBeActivatedOnTableWithSyncIndex) { + SenderShouldBeActivated("/Root/Table", TableWithIndex(SimpleSyncIndex())); + } + + Y_UNIT_TEST(SenderShouldBeActivatedOnTableWithAsyncIndex) { + SenderShouldBeActivated("/Root/Table", TableWithIndex(SimpleAsyncIndex())); + } + + void SenderShouldShakeHands(const TString& path, ui32 times, const TShardedTableOptions& opts, + TMaybe<TShardedTableOptions::TIndex> addIndex = Nothing()) + { + const auto pathParts = SplitPath(path); + UNIT_ASSERT(pathParts.size() > 1); + + const auto domainName = pathParts.at(0); + const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); + const auto tableName = pathParts.at(pathParts.size() - 1); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName(domainName) + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + ui32 counter = 0; + runtime.SetObserverFunc([&counter](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvChangeExchange::EvHandshake) { + ++counter; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + CreateShardedTable(server, sender, workingDir, tableName, opts); + if (addIndex) { + AsyncAlterAddIndex(server, domainName, path, *addIndex); + } + + if (counter != times) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&](IEventHandle&) { + return counter == times; + }); + server->GetRuntime()->DispatchEvents(opts); + } + } + + Y_UNIT_TEST(SenderShouldShakeHandsOnce) { + SenderShouldShakeHands("/Root/Table", 1, TableWithIndex(SimpleAsyncIndex())); + } + + Y_UNIT_TEST(SenderShouldShakeHandsTwice) { + SenderShouldShakeHands("/Root/Table", 2, TShardedTableOptions() + .Columns({ {"pkey", "Uint32", true, false}, {"i1key", "Uint32", false, false}, {"i2key", "Uint32", false, false}, - }) - .Indexes({ + }) + .Indexes({ {"by_i1key", {"i1key"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, {"by_i2key", {"i2key"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - }) - ); - } - - Y_UNIT_TEST(SenderShouldShakeHandsAfterAddingIndex) { - SenderShouldShakeHands("/Root/Table", 1, TableWoIndexes(), SimpleAsyncIndex()); - } - - void ShouldDeliverChanges(const TString& path, const TShardedTableOptions& opts, - TMaybe<TShardedTableOptions::TIndex> addIndex, const TVector<TString>& queries) - { - const auto pathParts = SplitPath(path); - UNIT_ASSERT(pathParts.size() > 1); - - const auto domainName = pathParts.at(0); - const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); - const auto tableName = pathParts.at(pathParts.size() - 1); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName(domainName) - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - THashSet<ui64> enqueued; - THashSet<ui64> requested; - THashSet<ui64> removed; - - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::EvEnqueueRecords: - for (const auto& record : ev->Get<TEvChangeExchange::TEvEnqueueRecords>()->Records) { - enqueued.insert(record.Order); - } - break; - - case TEvChangeExchange::EvRequestRecords: - for (const auto& record : ev->Get<TEvChangeExchange::TEvRequestRecords>()->Records) { - requested.insert(record.Order); - } - break; - - case TEvChangeExchange::EvRemoveRecords: - for (const auto& record : ev->Get<TEvChangeExchange::TEvRemoveRecords>()->Records) { - removed.insert(record); - } - break; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - CreateShardedTable(server, sender, workingDir, tableName, opts); - if (addIndex) { - WaitTxNotification(server, sender, AsyncAlterAddIndex(server, domainName, path, *addIndex)); - } - - for (const auto& query : queries) { - ExecSQL(server, sender, query); - } - - if (removed.size() != queries.size()) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&removed, expected = queries.size()](IEventHandle&) { - return removed.size() == expected; - }); - server->GetRuntime()->DispatchEvents(opts); - } - - UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), requested.size()); - UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), removed.size()); - } - - Y_UNIT_TEST(ShouldDeliverChangesOnFreshTable) { - ShouldDeliverChanges("/Root/Table", TableWithIndex(SimpleAsyncIndex()), Nothing(), { - "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);", - "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);", - }); - } - - Y_UNIT_TEST(ShouldDeliverChangesOnAlteredTable) { - ShouldDeliverChanges("/Root/Table", TableWoIndexes(), SimpleAsyncIndex(), { - "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);", - "DELETE FROM `/Root/Table` WHERE pkey = 1;", - }); - } - - Y_UNIT_TEST(ShouldRemoveRecordsAfterDroppingIndex) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - bool preventActivation = true; - TVector<THolder<IEventHandle>> activations; - - THashSet<ui64> enqueued; - THashSet<ui64> removed; - - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::EvActivateSender: - if (preventActivation) { - activations.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } else { - return TTestActorRuntime::EEventAction::PROCESS; - } - - case TEvChangeExchange::EvEnqueueRecords: - for (const auto& record : ev->Get<TEvChangeExchange::TEvEnqueueRecords>()->Records) { - enqueued.insert(record.Order); - } - break; - - case TEvChangeExchange::EvRemoveRecords: - for (const auto& record : ev->Get<TEvChangeExchange::TEvRemoveRecords>()->Records) { - removed.insert(record); - } - break; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); - ExecSQL(server, sender, "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);"); - WaitTxNotification(server, sender, AsyncAlterDropIndex(server, "/Root", "Table", "by_ikey")); - - if (activations.size() != 2 /* main + index */) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&activations](IEventHandle&) { - return activations.size() == 2; - }); - server->GetRuntime()->DispatchEvents(opts); - } - - preventActivation = false; - for (auto& ev : activations) { - server->GetRuntime()->Send(ev.Release(), 0, true); - } - - if (removed.size() != 1) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&removed](IEventHandle&) { - return removed.size() == 1; - }); - server->GetRuntime()->DispatchEvents(opts); - } - - UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), removed.size()); - } - - void WaitForContent(TServer::TPtr server, const TString& tablePath, const TString& expected) { - while (true) { - auto content = ReadShardedTable(server, tablePath); - if (StripInPlace(content) == expected) { - break; - } else { - SimulateSleep(server, TDuration::Seconds(1)); - } - } - } - - Y_UNIT_TEST(ShouldDeliverChangesOnSplitMerge) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - bool preventEnqueueing = true; - TVector<THolder<IEventHandle>> enqueued; - THashMap<ui64, ui32> splitAcks; - - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::EvEnqueueRecords: - if (preventEnqueueing) { - enqueued.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } else { - return TTestActorRuntime::EEventAction::PROCESS; - } - - case TEvDataShard::EvSplitAck: - ++splitAcks[ev->Get<TEvDataShard::TEvSplitAck>()->Record.GetOperationCookie()]; - break; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - auto waitForSplitAcks = [&](ui64 txId, ui32 count) { - if (splitAcks[txId] != count) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&](IEventHandle&) { - return splitAcks[txId] == count; - }); - server->GetRuntime()->DispatchEvents(opts); - } - }; - - auto sendEnqueued = [&]() { - preventEnqueueing = false; - for (auto& ev : std::exchange(enqueued, TVector<THolder<IEventHandle>>())) { - server->GetRuntime()->Send(ev.Release(), 0, true); - } - }; - - CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); - SimulateSleep(server, TDuration::Seconds(1)); - SetSplitMergePartCountLimit(&runtime, -1); - - // split of main table - ExecSQL(server, sender, R"( - UPSERT INTO `/Root/Table` (pkey, ikey) VALUES - (1, 10), - (2, 20), - (3, 30); - )"); - - auto tabletIds = GetTableShards(server, sender, "/Root/Table"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - - auto txId = AsyncSplitTable(server, sender, "/Root/Table", tabletIds.at(0), 4); - waitForSplitAcks(txId, 1); - sendEnqueued(); - - WaitTxNotification(server, sender, txId); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 10, pkey = 1\nikey = 20, pkey = 2\nikey = 30, pkey = 3"); - - // merge of main table - preventEnqueueing = true; - ExecSQL(server, sender, R"( - UPSERT INTO `/Root/Table` (pkey, ikey) VALUES - (1, 11), - (2, 21), - (3, 31); - )"); - - tabletIds = GetTableShards(server, sender, "/Root/Table"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 2); - - txId = AsyncMergeTable(server, sender, "/Root/Table", tabletIds); - waitForSplitAcks(txId, 2); - - ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (3, 32);"); - sendEnqueued(); - - WaitTxNotification(server, sender, txId); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 11, pkey = 1\nikey = 21, pkey = 2\nikey = 32, pkey = 3"); - - // split of index table - preventEnqueueing = true; - ExecSQL(server, sender, R"( - UPSERT INTO `/Root/Table` (pkey, ikey) VALUES - (1, 13), - (2, 23), - (3, 33); - )"); - - tabletIds = GetTableShards(server, sender, "/Root/Table/by_ikey/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - - txId = AsyncSplitTable(server, sender, "/Root/Table/by_ikey/indexImplTable", tabletIds.at(0), 40); - waitForSplitAcks(txId, 1); - - ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (4, 44);"); - sendEnqueued(); - - WaitTxNotification(server, sender, txId); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 13, pkey = 1\nikey = 23, pkey = 2\nikey = 33, pkey = 3\nikey = 44, pkey = 4"); - - // merge of index table - preventEnqueueing = true; - ExecSQL(server, sender, R"( - UPSERT INTO `/Root/Table` (pkey, ikey) VALUES - (1, 15), - (2, 25), - (3, 35), - (4, 45); - )"); - - tabletIds = GetTableShards(server, sender, "/Root/Table/by_ikey/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 2); - - txId = AsyncMergeTable(server, sender, "/Root/Table/by_ikey/indexImplTable", tabletIds); - waitForSplitAcks(txId, 2); - sendEnqueued(); - - WaitTxNotification(server, sender, txId); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 15, pkey = 1\nikey = 25, pkey = 2\nikey = 35, pkey = 3\nikey = 45, pkey = 4"); - } - - using TSetQueueLimitFunc = std::function<void(TServerSettings&)>; - void ShouldRejectChangesOnQueueOverflow(TSetQueueLimitFunc setLimit) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") - .SetUseRealThreads(false) - .SetEnableDataColumnForIndexTable(true) - .SetEnableAsyncIndexes(true); - setLimit(serverSettings); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); - InitRoot(server, sender); - - bool preventEnqueueing = true; - TVector<THolder<IEventHandle>> enqueued; - - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvChangeExchange::EvEnqueueRecords: - if (preventEnqueueing) { - enqueued.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } else { - return TTestActorRuntime::EEventAction::PROCESS; - } - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - auto sendEnqueued = [&]() { - preventEnqueueing = false; - for (auto& ev : std::exchange(enqueued, TVector<THolder<IEventHandle>>())) { - server->GetRuntime()->Send(ev.Release(), 0, true); - } - }; - - CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); - - ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);"); - ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);", true, Ydb::StatusIds::OVERLOADED); - - sendEnqueued(); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 10, pkey = 1"); - - ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);"); - WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", - "ikey = 10, pkey = 1\nikey = 20, pkey = 2"); - } - - Y_UNIT_TEST(ShouldRejectChangesOnQueueOverflowByCount) { - ShouldRejectChangesOnQueueOverflow([](TServerSettings& opts) { - opts.SetChangesQueueItemsLimit(1); - }); - } - - Y_UNIT_TEST(ShouldRejectChangesOnQueueOverflowBySize) { - ShouldRejectChangesOnQueueOverflow([](TServerSettings& opts) { - opts.SetChangesQueueBytesLimit(1); - }); - } -} - -} // NKikimr + }) + ); + } + + Y_UNIT_TEST(SenderShouldShakeHandsAfterAddingIndex) { + SenderShouldShakeHands("/Root/Table", 1, TableWoIndexes(), SimpleAsyncIndex()); + } + + void ShouldDeliverChanges(const TString& path, const TShardedTableOptions& opts, + TMaybe<TShardedTableOptions::TIndex> addIndex, const TVector<TString>& queries) + { + const auto pathParts = SplitPath(path); + UNIT_ASSERT(pathParts.size() > 1); + + const auto domainName = pathParts.at(0); + const auto workingDir = CombinePath(pathParts.begin(), pathParts.begin() + pathParts.size() - 1); + const auto tableName = pathParts.at(pathParts.size() - 1); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName(domainName) + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + THashSet<ui64> enqueued; + THashSet<ui64> requested; + THashSet<ui64> removed; + + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::EvEnqueueRecords: + for (const auto& record : ev->Get<TEvChangeExchange::TEvEnqueueRecords>()->Records) { + enqueued.insert(record.Order); + } + break; + + case TEvChangeExchange::EvRequestRecords: + for (const auto& record : ev->Get<TEvChangeExchange::TEvRequestRecords>()->Records) { + requested.insert(record.Order); + } + break; + + case TEvChangeExchange::EvRemoveRecords: + for (const auto& record : ev->Get<TEvChangeExchange::TEvRemoveRecords>()->Records) { + removed.insert(record); + } + break; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + CreateShardedTable(server, sender, workingDir, tableName, opts); + if (addIndex) { + WaitTxNotification(server, sender, AsyncAlterAddIndex(server, domainName, path, *addIndex)); + } + + for (const auto& query : queries) { + ExecSQL(server, sender, query); + } + + if (removed.size() != queries.size()) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&removed, expected = queries.size()](IEventHandle&) { + return removed.size() == expected; + }); + server->GetRuntime()->DispatchEvents(opts); + } + + UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), requested.size()); + UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), removed.size()); + } + + Y_UNIT_TEST(ShouldDeliverChangesOnFreshTable) { + ShouldDeliverChanges("/Root/Table", TableWithIndex(SimpleAsyncIndex()), Nothing(), { + "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);", + "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);", + }); + } + + Y_UNIT_TEST(ShouldDeliverChangesOnAlteredTable) { + ShouldDeliverChanges("/Root/Table", TableWoIndexes(), SimpleAsyncIndex(), { + "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);", + "DELETE FROM `/Root/Table` WHERE pkey = 1;", + }); + } + + Y_UNIT_TEST(ShouldRemoveRecordsAfterDroppingIndex) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + bool preventActivation = true; + TVector<THolder<IEventHandle>> activations; + + THashSet<ui64> enqueued; + THashSet<ui64> removed; + + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::EvActivateSender: + if (preventActivation) { + activations.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } else { + return TTestActorRuntime::EEventAction::PROCESS; + } + + case TEvChangeExchange::EvEnqueueRecords: + for (const auto& record : ev->Get<TEvChangeExchange::TEvEnqueueRecords>()->Records) { + enqueued.insert(record.Order); + } + break; + + case TEvChangeExchange::EvRemoveRecords: + for (const auto& record : ev->Get<TEvChangeExchange::TEvRemoveRecords>()->Records) { + removed.insert(record); + } + break; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); + ExecSQL(server, sender, "INSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);"); + WaitTxNotification(server, sender, AsyncAlterDropIndex(server, "/Root", "Table", "by_ikey")); + + if (activations.size() != 2 /* main + index */) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&activations](IEventHandle&) { + return activations.size() == 2; + }); + server->GetRuntime()->DispatchEvents(opts); + } + + preventActivation = false; + for (auto& ev : activations) { + server->GetRuntime()->Send(ev.Release(), 0, true); + } + + if (removed.size() != 1) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&removed](IEventHandle&) { + return removed.size() == 1; + }); + server->GetRuntime()->DispatchEvents(opts); + } + + UNIT_ASSERT_VALUES_EQUAL(enqueued.size(), removed.size()); + } + + void WaitForContent(TServer::TPtr server, const TString& tablePath, const TString& expected) { + while (true) { + auto content = ReadShardedTable(server, tablePath); + if (StripInPlace(content) == expected) { + break; + } else { + SimulateSleep(server, TDuration::Seconds(1)); + } + } + } + + Y_UNIT_TEST(ShouldDeliverChangesOnSplitMerge) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + bool preventEnqueueing = true; + TVector<THolder<IEventHandle>> enqueued; + THashMap<ui64, ui32> splitAcks; + + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::EvEnqueueRecords: + if (preventEnqueueing) { + enqueued.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } else { + return TTestActorRuntime::EEventAction::PROCESS; + } + + case TEvDataShard::EvSplitAck: + ++splitAcks[ev->Get<TEvDataShard::TEvSplitAck>()->Record.GetOperationCookie()]; + break; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + auto waitForSplitAcks = [&](ui64 txId, ui32 count) { + if (splitAcks[txId] != count) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&](IEventHandle&) { + return splitAcks[txId] == count; + }); + server->GetRuntime()->DispatchEvents(opts); + } + }; + + auto sendEnqueued = [&]() { + preventEnqueueing = false; + for (auto& ev : std::exchange(enqueued, TVector<THolder<IEventHandle>>())) { + server->GetRuntime()->Send(ev.Release(), 0, true); + } + }; + + CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); + SimulateSleep(server, TDuration::Seconds(1)); + SetSplitMergePartCountLimit(&runtime, -1); + + // split of main table + ExecSQL(server, sender, R"( + UPSERT INTO `/Root/Table` (pkey, ikey) VALUES + (1, 10), + (2, 20), + (3, 30); + )"); + + auto tabletIds = GetTableShards(server, sender, "/Root/Table"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + + auto txId = AsyncSplitTable(server, sender, "/Root/Table", tabletIds.at(0), 4); + waitForSplitAcks(txId, 1); + sendEnqueued(); + + WaitTxNotification(server, sender, txId); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 10, pkey = 1\nikey = 20, pkey = 2\nikey = 30, pkey = 3"); + + // merge of main table + preventEnqueueing = true; + ExecSQL(server, sender, R"( + UPSERT INTO `/Root/Table` (pkey, ikey) VALUES + (1, 11), + (2, 21), + (3, 31); + )"); + + tabletIds = GetTableShards(server, sender, "/Root/Table"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 2); + + txId = AsyncMergeTable(server, sender, "/Root/Table", tabletIds); + waitForSplitAcks(txId, 2); + + ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (3, 32);"); + sendEnqueued(); + + WaitTxNotification(server, sender, txId); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 11, pkey = 1\nikey = 21, pkey = 2\nikey = 32, pkey = 3"); + + // split of index table + preventEnqueueing = true; + ExecSQL(server, sender, R"( + UPSERT INTO `/Root/Table` (pkey, ikey) VALUES + (1, 13), + (2, 23), + (3, 33); + )"); + + tabletIds = GetTableShards(server, sender, "/Root/Table/by_ikey/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + + txId = AsyncSplitTable(server, sender, "/Root/Table/by_ikey/indexImplTable", tabletIds.at(0), 40); + waitForSplitAcks(txId, 1); + + ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (4, 44);"); + sendEnqueued(); + + WaitTxNotification(server, sender, txId); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 13, pkey = 1\nikey = 23, pkey = 2\nikey = 33, pkey = 3\nikey = 44, pkey = 4"); + + // merge of index table + preventEnqueueing = true; + ExecSQL(server, sender, R"( + UPSERT INTO `/Root/Table` (pkey, ikey) VALUES + (1, 15), + (2, 25), + (3, 35), + (4, 45); + )"); + + tabletIds = GetTableShards(server, sender, "/Root/Table/by_ikey/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 2); + + txId = AsyncMergeTable(server, sender, "/Root/Table/by_ikey/indexImplTable", tabletIds); + waitForSplitAcks(txId, 2); + sendEnqueued(); + + WaitTxNotification(server, sender, txId); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 15, pkey = 1\nikey = 25, pkey = 2\nikey = 35, pkey = 3\nikey = 45, pkey = 4"); + } + + using TSetQueueLimitFunc = std::function<void(TServerSettings&)>; + void ShouldRejectChangesOnQueueOverflow(TSetQueueLimitFunc setLimit) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") + .SetUseRealThreads(false) + .SetEnableDataColumnForIndexTable(true) + .SetEnableAsyncIndexes(true); + setLimit(serverSettings); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::CHANGE_EXCHANGE, NLog::PRI_DEBUG); + InitRoot(server, sender); + + bool preventEnqueueing = true; + TVector<THolder<IEventHandle>> enqueued; + + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvChangeExchange::EvEnqueueRecords: + if (preventEnqueueing) { + enqueued.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } else { + return TTestActorRuntime::EEventAction::PROCESS; + } + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + auto sendEnqueued = [&]() { + preventEnqueueing = false; + for (auto& ev : std::exchange(enqueued, TVector<THolder<IEventHandle>>())) { + server->GetRuntime()->Send(ev.Release(), 0, true); + } + }; + + CreateShardedTable(server, sender, "/Root", "Table", TableWithIndex(SimpleAsyncIndex())); + + ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (1, 10);"); + ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);", true, Ydb::StatusIds::OVERLOADED); + + sendEnqueued(); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 10, pkey = 1"); + + ExecSQL(server, sender, "UPSERT INTO `/Root/Table` (pkey, ikey) VALUES (2, 20);"); + WaitForContent(server, "/Root/Table/by_ikey/indexImplTable", + "ikey = 10, pkey = 1\nikey = 20, pkey = 2"); + } + + Y_UNIT_TEST(ShouldRejectChangesOnQueueOverflowByCount) { + ShouldRejectChangesOnQueueOverflow([](TServerSettings& opts) { + opts.SetChangesQueueItemsLimit(1); + }); + } + + Y_UNIT_TEST(ShouldRejectChangesOnQueueOverflowBySize) { + ShouldRejectChangesOnQueueOverflow([](TServerSettings& opts) { + opts.SetChangesQueueBytesLimit(1); + }); + } +} + +} // NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index 2cde01573dd..d754f7ad0e2 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -41,20 +41,20 @@ void TTester::Setup(TTestActorRuntime& runtime, const TOptions& opts) { app.SetEnableMvcc(opts.Mvcc); - auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds( + auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds( "dc-1", domainId, FAKE_SCHEMESHARD_TABLET_ID, domainId, domainId, TVector<ui32>{domainId}, domainId, TVector<ui32>{domainId}, planResolution, TVector<ui64>{TDomainsInfo::MakeTxCoordinatorIDFixed(domainId, 1)}, TVector<ui64>{}, - TVector<ui64>{TDomainsInfo::MakeTxAllocatorIDFixed(domainId, 1)}); + TVector<ui64>{TDomainsInfo::MakeTxAllocatorIDFixed(domainId, 1)}); - TVector<ui64> ids = runtime.GetTxAllocatorTabletIds(); - ids.insert(ids.end(), domain->TxAllocators.begin(), domain->TxAllocators.end()); - runtime.SetTxAllocatorTabletIds(ids); - - app.AddDomain(domain.Release()); + TVector<ui64> ids = runtime.GetTxAllocatorTabletIds(); + ids.insert(ids.end(), domain->TxAllocators.begin(), domain->TxAllocators.end()); + runtime.SetTxAllocatorTabletIds(ids); + + app.AddDomain(domain.Release()); SetupTabletServices(runtime, &app); } @@ -1091,7 +1091,7 @@ void CreateShardedTable(Tests::TServer::TPtr server, TActorId sender, const TString &root, const TString &name, - const TShardedTableOptions &opts) + const TShardedTableOptions &opts) { auto &runtime = *server->GetRuntime(); auto &settings = server->GetSettings(); @@ -1104,56 +1104,56 @@ void CreateShardedTable(Tests::TServer::TPtr server, request->Record.SetExecTimeoutPeriod(Max<ui64>()); auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); tx.SetWorkingDir(root); - + NKikimrSchemeOp::TTableDescription* desc = nullptr; - if (opts.Indexes_) { + if (opts.Indexes_) { tx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateIndexedTable); - desc = tx.MutableCreateIndexedTable()->MutableTableDescription(); - } else { + desc = tx.MutableCreateIndexedTable()->MutableTableDescription(); + } else { tx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateTable); - desc = tx.MutableCreateTable(); - } - - desc->SetName(name); - - for (const auto& column : opts.Columns_) { - auto col = desc->AddColumns(); - col->SetName(column.Name); - col->SetType(column.Type); + desc = tx.MutableCreateTable(); + } + + desc->SetName(name); + + for (const auto& column : opts.Columns_) { + auto col = desc->AddColumns(); + col->SetName(column.Name); + col->SetType(column.Type); col->SetNotNull(column.NotNull); - if (column.IsKey) { - desc->AddKeyColumnNames(column.Name); - } + if (column.IsKey) { + desc->AddKeyColumnNames(column.Name); + } + } + + for (const auto& index : opts.Indexes_) { + auto* indexDesc = tx.MutableCreateIndexedTable()->MutableIndexDescription()->Add(); + + indexDesc->SetName(index.Name); + indexDesc->SetType(index.Type); + + for (const auto& col : index.IndexColumns) { + indexDesc->AddKeyColumnNames(col); + } + for (const auto& col : index.DataColumns) { + indexDesc->AddDataColumnNames(col); + } } - for (const auto& index : opts.Indexes_) { - auto* indexDesc = tx.MutableCreateIndexedTable()->MutableIndexDescription()->Add(); - - indexDesc->SetName(index.Name); - indexDesc->SetType(index.Type); - - for (const auto& col : index.IndexColumns) { - indexDesc->AddKeyColumnNames(col); - } - for (const auto& col : index.DataColumns) { - indexDesc->AddDataColumnNames(col); - } - } - - desc->SetUniformPartitionsCount(opts.Shards_); - - if (!opts.EnableOutOfOrder_) - desc->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(false); - - if (opts.Policy_) { - opts.Policy_->Serialize(*desc->MutablePartitionConfig()->MutableCompactionPolicy()); + desc->SetUniformPartitionsCount(opts.Shards_); + + if (!opts.EnableOutOfOrder_) + desc->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(false); + + if (opts.Policy_) { + opts.Policy_->Serialize(*desc->MutablePartitionConfig()->MutableCompactionPolicy()); } - switch (opts.ShadowData_) { + switch (opts.ShadowData_) { case EShadowDataMode::Default: break; case EShadowDataMode::Enabled: - desc->MutablePartitionConfig()->SetShadowData(true); + desc->MutablePartitionConfig()->SetShadowData(true); break; } @@ -1182,23 +1182,23 @@ void CreateShardedTable(Tests::TServer::TPtr server, } } -void CreateShardedTable(Tests::TServer::TPtr server, +void CreateShardedTable(Tests::TServer::TPtr server, TActorId sender, - const TString &root, - const TString &name, - ui64 shards, - bool enableOutOfOrder, - const NLocalDb::TCompactionPolicy* policy, - EShadowDataMode shadowData) -{ - auto opts = TShardedTableOptions() - .Shards(shards) - .EnableOutOfOrder(enableOutOfOrder) - .Policy(policy) - .ShadowData(shadowData); - CreateShardedTable(server, sender, root, name, opts); -} - + const TString &root, + const TString &name, + ui64 shards, + bool enableOutOfOrder, + const NLocalDb::TCompactionPolicy* policy, + EShadowDataMode shadowData) +{ + auto opts = TShardedTableOptions() + .Shards(shards) + .EnableOutOfOrder(enableOutOfOrder) + .Policy(policy) + .ShadowData(shadowData); + CreateShardedTable(server, sender, root, name, opts); +} + NKikimrScheme::TEvDescribeSchemeResult DescribeTable(Tests::TServer::TPtr server, TActorId sender, const TString &path) @@ -1209,7 +1209,7 @@ NKikimrScheme::TEvDescribeSchemeResult DescribeTable(Tests::TServer::TPtr server auto request = MakeHolder<TEvTxUserProxy::TEvNavigate>(); request->Record.MutableDescribePath()->SetPath(path); - request->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); + request->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); auto reply = runtime.GrabEdgeEventRethrow<TEvSchemeShard::TEvDescribeSchemeResult>(handle); @@ -1263,7 +1263,7 @@ TTableId ResolveTableId( auto& entry = request->ResultSet.emplace_back(); entry.Path = SplitPath(path); entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; runtime.Send(new IEventHandle(MakeSchemeCacheID(), sender, new TEvTxProxySchemeCache::TEvNavigateKeySet(request))); } @@ -1573,121 +1573,121 @@ ui64 AsyncAlterAndDisableShadow( return ev->Get()->Record.GetTxId(); } -ui64 AsyncAlterAddIndex( - Tests::TServer::TPtr server, - const TString& dbName, - const TString& tablePath, - const TShardedTableOptions::TIndex& indexDesc) -{ - auto &runtime = *server->GetRuntime(); - auto &settings = server->GetSettings(); - auto sender = runtime.AllocateEdgeActor(); - - runtime.Send(new IEventHandle(MakeTxProxyID(), sender, new TEvTxUserProxy::TEvAllocateTxId())); - auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvAllocateTxIdResult>(sender); - const auto txId = ev->Get()->TxId; - - NKikimrIndexBuilder::TIndexBuildSettings buildSettings; - buildSettings.set_source_path(tablePath); - - Ydb::Table::TableIndex& index = *buildSettings.mutable_index(); - index.set_name(indexDesc.Name); - *index.mutable_index_columns() = {indexDesc.IndexColumns.begin(), indexDesc.IndexColumns.end()}; - *index.mutable_data_columns() = {indexDesc.DataColumns.begin(), indexDesc.DataColumns.end()}; - - switch (indexDesc.Type) { +ui64 AsyncAlterAddIndex( + Tests::TServer::TPtr server, + const TString& dbName, + const TString& tablePath, + const TShardedTableOptions::TIndex& indexDesc) +{ + auto &runtime = *server->GetRuntime(); + auto &settings = server->GetSettings(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, new TEvTxUserProxy::TEvAllocateTxId())); + auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvAllocateTxIdResult>(sender); + const auto txId = ev->Get()->TxId; + + NKikimrIndexBuilder::TIndexBuildSettings buildSettings; + buildSettings.set_source_path(tablePath); + + Ydb::Table::TableIndex& index = *buildSettings.mutable_index(); + index.set_name(indexDesc.Name); + *index.mutable_index_columns() = {indexDesc.IndexColumns.begin(), indexDesc.IndexColumns.end()}; + *index.mutable_data_columns() = {indexDesc.DataColumns.begin(), indexDesc.DataColumns.end()}; + + switch (indexDesc.Type) { case NKikimrSchemeOp::EIndexTypeGlobal: - *index.mutable_global_index() = Ydb::Table::GlobalIndex(); - break; + *index.mutable_global_index() = Ydb::Table::GlobalIndex(); + break; case NKikimrSchemeOp::EIndexTypeGlobalAsync: - *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; - default: - UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(indexDesc.Type)); - } - - auto req = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(txId, dbName, std::move(buildSettings)); - auto tabletId = ChangeStateStorage(SchemeRoot, settings.Domain); - runtime.SendToPipe(tabletId, sender, req.Release(), 0, GetPipeConfigWithRetries()); - - auto resp = runtime.GrabEdgeEventRethrow<TEvIndexBuilder::TEvCreateResponse>(sender); - UNIT_ASSERT_EQUAL(resp->Get()->Record.GetStatus(), Ydb::StatusIds::SUCCESS); - return txId; -} - -ui64 AsyncAlterDropIndex( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TString& indexName) -{ - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - - auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetExecTimeoutPeriod(Max<ui64>()); - auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + break; + default: + UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(indexDesc.Type)); + } + + auto req = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(txId, dbName, std::move(buildSettings)); + auto tabletId = ChangeStateStorage(SchemeRoot, settings.Domain); + runtime.SendToPipe(tabletId, sender, req.Release(), 0, GetPipeConfigWithRetries()); + + auto resp = runtime.GrabEdgeEventRethrow<TEvIndexBuilder::TEvCreateResponse>(sender); + UNIT_ASSERT_EQUAL(resp->Get()->Record.GetStatus(), Ydb::StatusIds::SUCCESS); + return txId; +} + +ui64 AsyncAlterDropIndex( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TString& indexName) +{ + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); tx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropIndex); - tx.SetWorkingDir(workingDir); - auto &desc = *tx.MutableDropIndex(); - desc.SetTableName(tableName); - desc.SetIndexName(indexName); - - runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); - auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); - return ev->Get()->Record.GetTxId(); -} - -ui64 AsyncAlterAddStream( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TShardedTableOptions::TCdcStream& streamDesc) -{ - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - - auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetExecTimeoutPeriod(Max<ui64>()); - auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + tx.SetWorkingDir(workingDir); + auto &desc = *tx.MutableDropIndex(); + desc.SetTableName(tableName); + desc.SetIndexName(indexName); + + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); + return ev->Get()->Record.GetTxId(); +} + +ui64 AsyncAlterAddStream( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TShardedTableOptions::TCdcStream& streamDesc) +{ + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); tx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateCdcStream); - tx.SetWorkingDir(workingDir); - auto &desc = *tx.MutableCreateCdcStream(); - desc.SetTableName(tableName); - desc.MutableStreamDescription()->SetName(streamDesc.Name); - desc.MutableStreamDescription()->SetMode(streamDesc.Mode); - - runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); - auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); - return ev->Get()->Record.GetTxId(); -} - -ui64 AsyncAlterDropStream( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TString& streamName) -{ - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - - auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetExecTimeoutPeriod(Max<ui64>()); - auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + tx.SetWorkingDir(workingDir); + auto &desc = *tx.MutableCreateCdcStream(); + desc.SetTableName(tableName); + desc.MutableStreamDescription()->SetName(streamDesc.Name); + desc.MutableStreamDescription()->SetMode(streamDesc.Mode); + + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); + return ev->Get()->Record.GetTxId(); +} + +ui64 AsyncAlterDropStream( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TString& streamName) +{ + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); tx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropCdcStream); - tx.SetWorkingDir(workingDir); - auto &desc = *tx.MutableDropCdcStream(); - desc.SetTableName(tableName); - desc.SetStreamName(streamName); - - runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); - auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); - return ev->Get()->Record.GetTxId(); -} - + tx.SetWorkingDir(workingDir); + auto &desc = *tx.MutableDropCdcStream(); + desc.SetTableName(tableName); + desc.SetStreamName(streamName); + + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress); + return ev->Get()->Record.GetTxId(); +} + void WaitTxNotification(Tests::TServer::TPtr server, TActorId sender, ui64 txId) { auto &runtime = *server->GetRuntime(); auto &settings = server->GetSettings(); @@ -1844,7 +1844,7 @@ namespace { Result << ", "; } Result << columns[idx].Name << " = "; - PrintValue(Result, parser.ColumnParser(idx)); + PrintValue(Result, parser.ColumnParser(idx)); } Result << Endl; } @@ -1909,49 +1909,49 @@ namespace { } } - static void PrintValue(TStringBuilder& out, NYdb::TValueParser& parser) { - switch (parser.GetKind()) { - case NYdb::TTypeParser::ETypeKind::Optional: - parser.OpenOptional(); - if (parser.IsNull()) { - out << "(empty maybe)"; - } else { - PrintValue(out, parser); - } - parser.CloseOptional(); - break; - - case NYdb::TTypeParser::ETypeKind::Primitive: - PrintPrimitive(out, parser); - break; - - default: - Y_FAIL("Unhandled"); - } - } - - static void PrintPrimitive(TStringBuilder& out, const NYdb::TValueParser& parser) { - #define PRINT_PRIMITIVE(type) \ - case NYdb::EPrimitiveType::type: \ - out << parser.Get##type(); \ - break - - switch (parser.GetPrimitiveType()) { - PRINT_PRIMITIVE(Uint32); - PRINT_PRIMITIVE(Uint64); - PRINT_PRIMITIVE(Date); - PRINT_PRIMITIVE(Datetime); - PRINT_PRIMITIVE(Timestamp); - PRINT_PRIMITIVE(String); - PRINT_PRIMITIVE(DyNumber); - - default: - Y_FAIL("Unhandled"); - } - - #undef PRINT_PRIMITIVE - } - + static void PrintValue(TStringBuilder& out, NYdb::TValueParser& parser) { + switch (parser.GetKind()) { + case NYdb::TTypeParser::ETypeKind::Optional: + parser.OpenOptional(); + if (parser.IsNull()) { + out << "(empty maybe)"; + } else { + PrintValue(out, parser); + } + parser.CloseOptional(); + break; + + case NYdb::TTypeParser::ETypeKind::Primitive: + PrintPrimitive(out, parser); + break; + + default: + Y_FAIL("Unhandled"); + } + } + + static void PrintPrimitive(TStringBuilder& out, const NYdb::TValueParser& parser) { + #define PRINT_PRIMITIVE(type) \ + case NYdb::EPrimitiveType::type: \ + out << parser.Get##type(); \ + break + + switch (parser.GetPrimitiveType()) { + PRINT_PRIMITIVE(Uint32); + PRINT_PRIMITIVE(Uint64); + PRINT_PRIMITIVE(Date); + PRINT_PRIMITIVE(Datetime); + PRINT_PRIMITIVE(Timestamp); + PRINT_PRIMITIVE(String); + PRINT_PRIMITIVE(DyNumber); + + default: + Y_FAIL("Unhandled"); + } + + #undef PRINT_PRIMITIVE + } + private: struct TPendingRequest { TActorId Sender; diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h index 8a7cdf4fa17..4b5d7fe4ea0 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.h +++ b/ydb/core/tx/datashard/datashard_ut_common.h @@ -21,7 +21,7 @@ class TFakeMiniKQLProxy; class TFakeProxyTx; constexpr ui64 FAKE_SCHEMESHARD_TABLET_ID = 4200; -constexpr ui64 FAKE_TX_ALLOCATOR_TABLET_ID = 4201; +constexpr ui64 FAKE_TX_ALLOCATOR_TABLET_ID = 4201; /// class TTester : public TNonCopyable { @@ -367,61 +367,61 @@ enum class EShadowDataMode { Enabled, }; -struct TShardedTableOptions { - using TSelf = TShardedTableOptions; - - struct TColumn { +struct TShardedTableOptions { + using TSelf = TShardedTableOptions; + + struct TColumn { TColumn(const TString& name, const TString& type, bool isKey, bool notNull) : Name(name) , Type(type) , IsKey(isKey) , NotNull(notNull) {} - TString Name; - TString Type; - bool IsKey; + TString Name; + TString Type; + bool IsKey; bool NotNull; - }; - - struct TIndex { + }; + + struct TIndex { using EType = NKikimrSchemeOp::EIndexType; - - TString Name; - TVector<TString> IndexColumns; - TVector<TString> DataColumns = {}; - EType Type = EType::EIndexTypeGlobal; - }; - - struct TCdcStream { + + TString Name; + TVector<TString> IndexColumns; + TVector<TString> DataColumns = {}; + EType Type = EType::EIndexTypeGlobal; + }; + + struct TCdcStream { using EMode = NKikimrSchemeOp::ECdcStreamMode; - - TString Name; - EMode Mode; - }; - + + TString Name; + EMode Mode; + }; + #define TABLE_OPTION_IMPL(type, name, defaultValue) \ TSelf& name(type value) {\ name##_ = std::move(value); \ - return *this; \ - } \ - type name##_ = defaultValue - + return *this; \ + } \ + type name##_ = defaultValue + #define TABLE_OPTION(type, name, defaultValue) TABLE_OPTION_IMPL(type, name, defaultValue) - - TABLE_OPTION(ui64, Shards, 1); - TABLE_OPTION(bool, EnableOutOfOrder, true); + + TABLE_OPTION(ui64, Shards, 1); + TABLE_OPTION(bool, EnableOutOfOrder, true); TABLE_OPTION(const NLocalDb::TCompactionPolicy*, Policy, nullptr); - TABLE_OPTION(EShadowDataMode, ShadowData, EShadowDataMode::Default); + TABLE_OPTION(EShadowDataMode, ShadowData, EShadowDataMode::Default); TABLE_OPTION(TVector<TColumn>, Columns, (TVector<TColumn>{{"key", "Uint32", true, false}, {"value", "Uint32", false, false}})); - TABLE_OPTION(TVector<TIndex>, Indexes, {}); + TABLE_OPTION(TVector<TIndex>, Indexes, {}); TABLE_OPTION(ui64, Followers, 0); TABLE_OPTION(bool, FollowerPromotion, false); TABLE_OPTION(bool, ExternalStorage, false); - -#undef TABLE_OPTION -#undef TABLE_OPTION_IMPL -}; - + +#undef TABLE_OPTION +#undef TABLE_OPTION_IMPL +}; + #define Y_UNIT_TEST_WITH_MVCC_IMPL(N, OPT) \ template<bool OPT> void N(NUnitTest::TTestContext&); \ struct TTestRegistration##N { \ @@ -455,12 +455,12 @@ void CreateShardedTable(Tests::TServer::TPtr server, TActorId sender, const TString &root, const TString &name, - const TShardedTableOptions &opts = TShardedTableOptions()); - -void CreateShardedTable(Tests::TServer::TPtr server, + const TShardedTableOptions &opts = TShardedTableOptions()); + +void CreateShardedTable(Tests::TServer::TPtr server, TActorId sender, - const TString &root, - const TString &name, + const TString &root, + const TString &name, ui64 shards, bool enableOutOfOrder = true, const NLocalDb::TCompactionPolicy* policy = nullptr, @@ -550,30 +550,30 @@ ui64 AsyncAlterAndDisableShadow( const TString& name, const NLocalDb::TCompactionPolicy* policy = nullptr); -ui64 AsyncAlterAddIndex( - Tests::TServer::TPtr server, - const TString& dbName, - const TString& tablePath, - const TShardedTableOptions::TIndex& indexDesc); - -ui64 AsyncAlterDropIndex( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TString& indexName); - -ui64 AsyncAlterAddStream( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TShardedTableOptions::TCdcStream& streamDesc); - -ui64 AsyncAlterDropStream( - Tests::TServer::TPtr server, - const TString& workingDir, - const TString& tableName, - const TString& streamName); - +ui64 AsyncAlterAddIndex( + Tests::TServer::TPtr server, + const TString& dbName, + const TString& tablePath, + const TShardedTableOptions::TIndex& indexDesc); + +ui64 AsyncAlterDropIndex( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TString& indexName); + +ui64 AsyncAlterAddStream( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TShardedTableOptions::TCdcStream& streamDesc); + +ui64 AsyncAlterDropStream( + Tests::TServer::TPtr server, + const TString& workingDir, + const TString& tableName, + const TString& streamName); + struct TReadShardedTableState { TActorId Sender; TActorId Worker; diff --git a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp index 32ac4dc8ac5..87a51ef80ab 100644 --- a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp @@ -1,1767 +1,1767 @@ -#include "defs.h" -#include "datashard_distributed_erase.h" -#include "datashard_ut_common.h" -#include "datashard_ut_common_kqp.h" - +#include "defs.h" +#include "datashard_distributed_erase.h" +#include "datashard_ut_common.h" +#include "datashard_ut_common_kqp.h" + #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/tx.h> - -#include <util/generic/bitmap.h> -#include <util/string/printf.h> -#include <util/string/strip.h> - -namespace NKikimr { - + +#include <util/generic/bitmap.h> +#include <util/string/printf.h> +#include <util/string/strip.h> + +namespace NKikimr { + using namespace NDataShard::NKqpHelpers; -using namespace Tests; - -struct TProto { - using TEvEraseRequest = NKikimrTxDataShard::TEvEraseRowsRequest; - using TEvEraseResponse = NKikimrTxDataShard::TEvEraseRowsResponse; - - using TEvCondEraseRequest = NKikimrTxDataShard::TEvConditionalEraseRowsRequest; - using TEvCondEraseResponse = NKikimrTxDataShard::TEvConditionalEraseRowsResponse; - - using TLimits = TEvCondEraseRequest::TLimits; - - using TEvProposeTx = NKikimrTxDataShard::TEvProposeTransaction; - using TEvProposeTxResult = NKikimrTxDataShard::TEvProposeTransactionResult; - - using TDistributedEraseTx = NKikimrTxDataShard::TDistributedEraseTransaction; -}; - +using namespace Tests; + +struct TProto { + using TEvEraseRequest = NKikimrTxDataShard::TEvEraseRowsRequest; + using TEvEraseResponse = NKikimrTxDataShard::TEvEraseRowsResponse; + + using TEvCondEraseRequest = NKikimrTxDataShard::TEvConditionalEraseRowsRequest; + using TEvCondEraseResponse = NKikimrTxDataShard::TEvConditionalEraseRowsResponse; + + using TLimits = TEvCondEraseRequest::TLimits; + + using TEvProposeTx = NKikimrTxDataShard::TEvProposeTransaction; + using TEvProposeTxResult = NKikimrTxDataShard::TEvProposeTransactionResult; + + using TDistributedEraseTx = NKikimrTxDataShard::TDistributedEraseTransaction; +}; + using EUnit = NKikimrSchemeOp::TTTLSettings::EUnit; -struct TUnit { +struct TUnit { static constexpr EUnit AUTO = NKikimrSchemeOp::TTTLSettings::UNIT_AUTO; static constexpr EUnit SECONDS = NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS; static constexpr EUnit MILLISECONDS = NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS; static constexpr EUnit MICROSECONDS = NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS; static constexpr EUnit NANOSECONDS = NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS; -}; - -namespace { - -void CreateTable(TServer::TPtr server, const TActorId& sender, const TString& root, +}; + +namespace { + +void CreateTable(TServer::TPtr server, const TActorId& sender, const TString& root, const TString& name, const TString& ttlColType = "Timestamp") { - auto opts = TShardedTableOptions() - .EnableOutOfOrder(false) - .Columns({ + auto opts = TShardedTableOptions() + .EnableOutOfOrder(false) + .Columns({ {"key", "Uint32", true, false}, {"value", ttlColType, false, false} - }); - CreateShardedTable(server, sender, root, name, opts); -} - -void CreateIndexedTable(TServer::TPtr server, const TActorId& sender, const TString& root, - const TString& name, const TString& ttlColType = "Timestamp") { - auto opts = TShardedTableOptions() - .EnableOutOfOrder(false) - .Columns({ + }); + CreateShardedTable(server, sender, root, name, opts); +} + +void CreateIndexedTable(TServer::TPtr server, const TActorId& sender, const TString& root, + const TString& name, const TString& ttlColType = "Timestamp") { + auto opts = TShardedTableOptions() + .EnableOutOfOrder(false) + .Columns({ {"key", "Uint32", true, false}, {"skey", "Uint32", false, false}, {"tkey", "Uint32", false, false}, {"value", ttlColType, false, false} - }) - .Indexes({ - {"by_skey", {"skey"}}, - {"by_tkey", {"tkey"}} - }); - CreateShardedTable(server, sender, root, name, opts); -} - -using TKeyCellsMaker = std::function<TVector<TCell>(ui32)>; - -TVector<TCell> MakeKeyCells(ui32 key) { - return {TCell::Make(key)}; -} - -TVector<TCell> KeyCellsRepeater(ui32 key) { - return {TCell::Make(key), TCell::Make(key)}; -}; - -TVector<TString> SerializeKeys(const TVector<ui32>& keys, TKeyCellsMaker makeKeyCells = &MakeKeyCells) { - TVector<TString> serializedKeys; - - for (const ui32 key : keys) { - serializedKeys.emplace_back(TSerializedCellVec::Serialize(makeKeyCells(key))); - } - - return serializedKeys; -} - -TProto::TEvEraseRequest MakeEraseRowsRequest( - const TTableId& tableId, - const TVector<ui32>& keyTags, - const TVector<TString>& keys) { - - TProto::TEvEraseRequest request; - - request.SetTableId(tableId.PathId.LocalPathId); - request.SetSchemaVersion(tableId.SchemaVersion); - - for (const ui32 tag : keyTags) { - request.AddKeyColumnIds(tag); - } - - for (const auto& key : keys) { - request.AddKeyColumns(key); - } - - return request; -} - -template <typename TEvResponse, typename TDerived> -class TRequestRunner: public TActorBootstrapped<TDerived> { - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - if (ev->Get()->Status == NKikimrProto::OK) { - return; - } - - Bootstrap(); - } - -protected: - void Reply(typename TEvResponse::TPtr& ev) { - TlsActivationContext->AsActorContext().Send(ev->Forward(ReplyTo)); - } - - virtual void Handle(typename TEvResponse::TPtr& ev) { - Reply(ev); - PassAway(); - } - - void PassAway() override { - if (Pipe) { - NTabletPipe::CloseAndForgetClient(this->SelfId(), Pipe); - } - - IActor::PassAway(); - } - - virtual IEventBase* MakeRequest() const = 0; - -public: + }) + .Indexes({ + {"by_skey", {"skey"}}, + {"by_tkey", {"tkey"}} + }); + CreateShardedTable(server, sender, root, name, opts); +} + +using TKeyCellsMaker = std::function<TVector<TCell>(ui32)>; + +TVector<TCell> MakeKeyCells(ui32 key) { + return {TCell::Make(key)}; +} + +TVector<TCell> KeyCellsRepeater(ui32 key) { + return {TCell::Make(key), TCell::Make(key)}; +}; + +TVector<TString> SerializeKeys(const TVector<ui32>& keys, TKeyCellsMaker makeKeyCells = &MakeKeyCells) { + TVector<TString> serializedKeys; + + for (const ui32 key : keys) { + serializedKeys.emplace_back(TSerializedCellVec::Serialize(makeKeyCells(key))); + } + + return serializedKeys; +} + +TProto::TEvEraseRequest MakeEraseRowsRequest( + const TTableId& tableId, + const TVector<ui32>& keyTags, + const TVector<TString>& keys) { + + TProto::TEvEraseRequest request; + + request.SetTableId(tableId.PathId.LocalPathId); + request.SetSchemaVersion(tableId.SchemaVersion); + + for (const ui32 tag : keyTags) { + request.AddKeyColumnIds(tag); + } + + for (const auto& key : keys) { + request.AddKeyColumns(key); + } + + return request; +} + +template <typename TEvResponse, typename TDerived> +class TRequestRunner: public TActorBootstrapped<TDerived> { + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->Status == NKikimrProto::OK) { + return; + } + + Bootstrap(); + } + +protected: + void Reply(typename TEvResponse::TPtr& ev) { + TlsActivationContext->AsActorContext().Send(ev->Forward(ReplyTo)); + } + + virtual void Handle(typename TEvResponse::TPtr& ev) { + Reply(ev); + PassAway(); + } + + void PassAway() override { + if (Pipe) { + NTabletPipe::CloseAndForgetClient(this->SelfId(), Pipe); + } + + IActor::PassAway(); + } + + virtual IEventBase* MakeRequest() const = 0; + +public: explicit TRequestRunner(const TActorId& replyTo, ui64 tabletID) - : ReplyTo(replyTo) - , TabletID(tabletID) - { - } - - void Bootstrap() { - if (Pipe) { - NTabletPipe::CloseAndForgetClient(this->SelfId(), Pipe); - } - - Pipe = this->Register(NTabletPipe::CreateClient(this->SelfId(), TabletID)); - NTabletPipe::SendData(this->SelfId(), Pipe, this->MakeRequest()); - - this->Become(&TDerived::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - cFunc(TEvTabletPipe::TEvClientDestroyed::EventType, Bootstrap); - hFunc(TEvResponse, Handle); - } - } - - using TBase = TRequestRunner<TEvResponse, TDerived>; - -private: + : ReplyTo(replyTo) + , TabletID(tabletID) + { + } + + void Bootstrap() { + if (Pipe) { + NTabletPipe::CloseAndForgetClient(this->SelfId(), Pipe); + } + + Pipe = this->Register(NTabletPipe::CreateClient(this->SelfId(), TabletID)); + NTabletPipe::SendData(this->SelfId(), Pipe, this->MakeRequest()); + + this->Become(&TDerived::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + cFunc(TEvTabletPipe::TEvClientDestroyed::EventType, Bootstrap); + hFunc(TEvResponse, Handle); + } + } + + using TBase = TRequestRunner<TEvResponse, TDerived>; + +private: const TActorId ReplyTo; - const ui64 TabletID; - + const ui64 TabletID; + TActorId Pipe; -}; - -void EraseRows( +}; + +void EraseRows( TServer::TPtr server, const TActorId& sender, const TString& path, - const TTableId& tableId, TVector<ui32> keyTags, TVector<TString> keys, - ui32 status = TProto::TEvEraseResponse::OK, const TString& error = "") { - using TEvRequest = TEvDataShard::TEvEraseRowsRequest; - using TEvResponse = TEvDataShard::TEvEraseRowsResponse; - - class TEraser: public TRequestRunner<TEvResponse, TEraser> { - public: - explicit TEraser( - const TActorId& replyTo, ui64 tabletID, - const TTableId& tableId, TVector<ui32> keyTags, TVector<TString> keys) - : TBase(replyTo, tabletID) - , TableId(tableId) - , KeyTags(std::move(keyTags)) - , Keys(std::move(keys)) - { - } - - IEventBase* MakeRequest() const override { - auto request = MakeHolder<TEvRequest>(); - request->Record = MakeEraseRowsRequest(TableId, KeyTags, Keys); - return request.Release(); - } - - private: - const TTableId TableId; - const TVector<ui32> KeyTags; - const TVector<TString> Keys; - }; - - auto tabletIDs = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIDs.size(), 1); - server->GetRuntime()->Register(new TEraser(sender, tabletIDs[0], tableId, std::move(keyTags), std::move(keys))); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); - if (error) { - UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); - } -} - -void ConditionalEraseRows( + const TTableId& tableId, TVector<ui32> keyTags, TVector<TString> keys, + ui32 status = TProto::TEvEraseResponse::OK, const TString& error = "") { + using TEvRequest = TEvDataShard::TEvEraseRowsRequest; + using TEvResponse = TEvDataShard::TEvEraseRowsResponse; + + class TEraser: public TRequestRunner<TEvResponse, TEraser> { + public: + explicit TEraser( + const TActorId& replyTo, ui64 tabletID, + const TTableId& tableId, TVector<ui32> keyTags, TVector<TString> keys) + : TBase(replyTo, tabletID) + , TableId(tableId) + , KeyTags(std::move(keyTags)) + , Keys(std::move(keys)) + { + } + + IEventBase* MakeRequest() const override { + auto request = MakeHolder<TEvRequest>(); + request->Record = MakeEraseRowsRequest(TableId, KeyTags, Keys); + return request.Release(); + } + + private: + const TTableId TableId; + const TVector<ui32> KeyTags; + const TVector<TString> Keys; + }; + + auto tabletIDs = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIDs.size(), 1); + server->GetRuntime()->Register(new TEraser(sender, tabletIDs[0], tableId, std::move(keyTags), std::move(keys))); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); + if (error) { + UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); + } +} + +void ConditionalEraseRows( TServer::TPtr server, const TActorId& sender, const TString& path, - const TTableId& tableId, ui32 columnId, ui64 threshold, EUnit unit = TUnit::AUTO, + const TTableId& tableId, ui32 columnId, ui64 threshold, EUnit unit = TUnit::AUTO, const NDataShard::TIndexes& indexes = {}, const TProto::TLimits& limits = {}, - ui32 status = TProto::TEvCondEraseResponse::ACCEPTED, const TString& error = "") { - using TEvRequest = TEvDataShard::TEvConditionalEraseRowsRequest; - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - class TEraser: public TRequestRunner<TEvResponse, TEraser> { - protected: - void Handle(TEvResponse::TPtr& ev) override { - bool continue_ = false; - - switch (ev->Get()->Record.GetStatus()) { - case TEvResponse::ProtoRecordType::ACCEPTED: - case TEvResponse::ProtoRecordType::PARTIAL: - continue_ = true; - break; - - default: - break; - } - - Reply(ev); - - if (!continue_) { - PassAway(); - } - } - - public: - explicit TEraser( - const TActorId& replyTo, ui64 tabletID, - const TTableId& tableId, ui32 columnId, ui64 threshold, EUnit unit, - const TProto::TLimits& limits, + ui32 status = TProto::TEvCondEraseResponse::ACCEPTED, const TString& error = "") { + using TEvRequest = TEvDataShard::TEvConditionalEraseRowsRequest; + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + class TEraser: public TRequestRunner<TEvResponse, TEraser> { + protected: + void Handle(TEvResponse::TPtr& ev) override { + bool continue_ = false; + + switch (ev->Get()->Record.GetStatus()) { + case TEvResponse::ProtoRecordType::ACCEPTED: + case TEvResponse::ProtoRecordType::PARTIAL: + continue_ = true; + break; + + default: + break; + } + + Reply(ev); + + if (!continue_) { + PassAway(); + } + } + + public: + explicit TEraser( + const TActorId& replyTo, ui64 tabletID, + const TTableId& tableId, ui32 columnId, ui64 threshold, EUnit unit, + const TProto::TLimits& limits, const NDataShard::TIndexes& indexes) - : TBase(replyTo, tabletID) - , TableId(tableId) - , ColumnId(columnId) - , Threshold(threshold) - , ColumnUnit(unit) - , Limits(limits) - , Indexes(indexes) - { - } - - IEventBase* MakeRequest() const override { - auto request = MakeHolder<TEvRequest>(); - request->Record.SetTableId(TableId.PathId.LocalPathId); - request->Record.SetSchemaVersion(TableId.SchemaVersion); - request->Record.MutableExpiration()->SetColumnId(ColumnId); - request->Record.MutableExpiration()->SetWallClockTimestamp(Threshold); - request->Record.MutableExpiration()->SetColumnUnit(ColumnUnit); - request->Record.MutableLimits()->CopyFrom(Limits); - - for (const auto& [indexId, columnIds] : Indexes) { - auto& index = *request->Record.MutableIndexes()->Add(); - - index.SetOwnerId(indexId.PathId.OwnerId); - index.SetPathId(indexId.PathId.LocalPathId); - index.SetSchemaVersion(indexId.SchemaVersion); - - for (const auto [indexColumnId, mainColumnId] : columnIds) { - auto& keyMap = *index.MutableKeyMap()->Add(); - keyMap.SetIndexColumnId(indexColumnId); - keyMap.SetMainColumnId(mainColumnId); - } - } - - return request.Release(); - } - - private: - const TTableId TableId; - const ui32 ColumnId; - const ui64 Threshold; - const EUnit ColumnUnit; - const TProto::TLimits Limits; + : TBase(replyTo, tabletID) + , TableId(tableId) + , ColumnId(columnId) + , Threshold(threshold) + , ColumnUnit(unit) + , Limits(limits) + , Indexes(indexes) + { + } + + IEventBase* MakeRequest() const override { + auto request = MakeHolder<TEvRequest>(); + request->Record.SetTableId(TableId.PathId.LocalPathId); + request->Record.SetSchemaVersion(TableId.SchemaVersion); + request->Record.MutableExpiration()->SetColumnId(ColumnId); + request->Record.MutableExpiration()->SetWallClockTimestamp(Threshold); + request->Record.MutableExpiration()->SetColumnUnit(ColumnUnit); + request->Record.MutableLimits()->CopyFrom(Limits); + + for (const auto& [indexId, columnIds] : Indexes) { + auto& index = *request->Record.MutableIndexes()->Add(); + + index.SetOwnerId(indexId.PathId.OwnerId); + index.SetPathId(indexId.PathId.LocalPathId); + index.SetSchemaVersion(indexId.SchemaVersion); + + for (const auto [indexColumnId, mainColumnId] : columnIds) { + auto& keyMap = *index.MutableKeyMap()->Add(); + keyMap.SetIndexColumnId(indexColumnId); + keyMap.SetMainColumnId(mainColumnId); + } + } + + return request.Release(); + } + + private: + const TTableId TableId; + const ui32 ColumnId; + const ui64 Threshold; + const EUnit ColumnUnit; + const TProto::TLimits Limits; const NDataShard::TIndexes Indexes; - }; - - auto tabletIDs = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIDs.size(), 1); - server->GetRuntime()->Register(new TEraser(sender, tabletIDs[0], tableId, columnId, threshold, unit, limits, indexes)); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); - if (error) { - UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); - } -} - -void DistributedEraseTx( - TServer::TPtr server, const TActorId& sender, const TString& path, - ui64 txId, const TProto::TDistributedEraseTx& tx, - ui32 status = TProto::TEvProposeTxResult::PREPARED, const TString& error = "") { - using TEvRequest = TEvDataShard::TEvProposeTransaction; - using TEvResponse = TEvDataShard::TEvProposeTransactionResult; - - class TEraser: public TRequestRunner<TEvResponse, TEraser> { - public: - explicit TEraser( - const TActorId& replyTo, ui64 tabletId, - ui64 txId, const TProto::TDistributedEraseTx& tx) - : TBase(replyTo, tabletId) - , TxId(txId) - , Tx(tx) - { - } - - IEventBase* MakeRequest() const override { - return new TEvRequest( - NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE, SelfId(), TxId, Tx.SerializeAsString() - ); - } - - private: - const ui64 TxId; - const TProto::TDistributedEraseTx Tx; - }; - - auto tabletIds = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - server->GetRuntime()->Register(new TEraser(sender, tabletIds[0], txId, tx)); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); - if (error) { - UNIT_ASSERT(ev->Get()->Record.ErrorSize()); - UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetError(0).GetReason(), error); - } -} - -} // anonymous - -Y_UNIT_TEST_SUITE(EraseRowsTests) { + }; + + auto tabletIDs = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIDs.size(), 1); + server->GetRuntime()->Register(new TEraser(sender, tabletIDs[0], tableId, columnId, threshold, unit, limits, indexes)); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); + if (error) { + UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); + } +} + +void DistributedEraseTx( + TServer::TPtr server, const TActorId& sender, const TString& path, + ui64 txId, const TProto::TDistributedEraseTx& tx, + ui32 status = TProto::TEvProposeTxResult::PREPARED, const TString& error = "") { + using TEvRequest = TEvDataShard::TEvProposeTransaction; + using TEvResponse = TEvDataShard::TEvProposeTransactionResult; + + class TEraser: public TRequestRunner<TEvResponse, TEraser> { + public: + explicit TEraser( + const TActorId& replyTo, ui64 tabletId, + ui64 txId, const TProto::TDistributedEraseTx& tx) + : TBase(replyTo, tabletId) + , TxId(txId) + , Tx(tx) + { + } + + IEventBase* MakeRequest() const override { + return new TEvRequest( + NKikimrTxDataShard::TX_KIND_DISTRIBUTED_ERASE, SelfId(), TxId, Tx.SerializeAsString() + ); + } + + private: + const ui64 TxId; + const TProto::TDistributedEraseTx Tx; + }; + + auto tabletIds = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + server->GetRuntime()->Register(new TEraser(sender, tabletIds[0], txId, tx)); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); + if (error) { + UNIT_ASSERT(ev->Get()->Record.ErrorSize()); + UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetError(0).GetReason(), error); + } +} + +} // anonymous + +Y_UNIT_TEST_SUITE(EraseRowsTests) { void EraseRowsShouldSuccess(TMaybe<ui64> injectSchemaVersion, bool enableMvcc) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(enableMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + CreateTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, value) VALUES - (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto testTableId = TTableId(tableId.PathId, injectSchemaVersion.GetOrElse(tableId.SchemaVersion)); - EraseRows(server, sender, "/Root/table-1", testTableId, {1}, SerializeKeys({1, 2})); - - auto content = ReadShardedTable(server, "/Root/table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, value) VALUES + (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto testTableId = TTableId(tableId.PathId, injectSchemaVersion.GetOrElse(tableId.SchemaVersion)); + EraseRows(server, sender, "/Root/table-1", testTableId, {1}, SerializeKeys({1, 2})); + + auto content = ReadShardedTable(server, "/Root/table-1"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, value = 2020-04-15T00:00:00.000000Z"); - } - + } + Y_UNIT_TEST_WITH_MVCC(EraseRowsShouldSuccess) { EraseRowsShouldSuccess(Nothing(), WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(EraseRowsShouldFailOnVariousErrors) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(WithMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + CreateTable(server, sender, "/Root", "table-1"); - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - - EraseRows(server, sender, "/Root/table-1", TTableId(), {1}, SerializeKeys({1, 2}), - TProto::TEvEraseResponse::SCHEME_ERROR, "Unknown table id"); - - EraseRows(server, sender, "/Root/table-1", TTableId(tableId.PathId, 100), {1}, SerializeKeys({1, 2}), - TProto::TEvEraseResponse::SCHEME_ERROR, "Schema version mismatch"); - - EraseRows(server, sender, "/Root/table-1", tableId, {0, 1}, SerializeKeys({1, 2}), - TProto::TEvEraseResponse::SCHEME_ERROR, "Key column count mismatch"); - - EraseRows(server, sender, "/Root/table-1", tableId, {0}, SerializeKeys({1, 2}), - TProto::TEvEraseResponse::SCHEME_ERROR, "Key column schema mismatch"); - - EraseRows(server, sender, "/Root/table-1", tableId, {1}, {"trash"}, - TProto::TEvEraseResponse::BAD_REQUEST, "Cannot parse key"); - - EraseRows(server, sender, "/Root/table-1", tableId, {1}, SerializeKeys({1, 2}, &KeyCellsRepeater), - TProto::TEvEraseResponse::SCHEME_ERROR, "Cell count doesn't match row scheme"); - } - + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + + EraseRows(server, sender, "/Root/table-1", TTableId(), {1}, SerializeKeys({1, 2}), + TProto::TEvEraseResponse::SCHEME_ERROR, "Unknown table id"); + + EraseRows(server, sender, "/Root/table-1", TTableId(tableId.PathId, 100), {1}, SerializeKeys({1, 2}), + TProto::TEvEraseResponse::SCHEME_ERROR, "Schema version mismatch"); + + EraseRows(server, sender, "/Root/table-1", tableId, {0, 1}, SerializeKeys({1, 2}), + TProto::TEvEraseResponse::SCHEME_ERROR, "Key column count mismatch"); + + EraseRows(server, sender, "/Root/table-1", tableId, {0}, SerializeKeys({1, 2}), + TProto::TEvEraseResponse::SCHEME_ERROR, "Key column schema mismatch"); + + EraseRows(server, sender, "/Root/table-1", tableId, {1}, {"trash"}, + TProto::TEvEraseResponse::BAD_REQUEST, "Cannot parse key"); + + EraseRows(server, sender, "/Root/table-1", tableId, {1}, SerializeKeys({1, 2}, &KeyCellsRepeater), + TProto::TEvEraseResponse::SCHEME_ERROR, "Cell count doesn't match row scheme"); + } + void ConditionalEraseShouldSuccess(const TString& ttlColType, EUnit unit, const TString& toUpload, const TString& afterErase, bool enableMvcc = false) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(enableMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + CreateTable(server, sender, "/Root", "table-1", ttlColType); - ExecSQL(server, sender, toUpload); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue(), unit); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - - auto content = ReadShardedTable(server, "/Root/table-1"); + ExecSQL(server, sender, toUpload); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue(), unit); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + + auto content = ReadShardedTable(server, "/Root/table-1"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), Strip(afterErase)); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldErase) { - ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), -(2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), -(3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(4, NULL); - )", R"( -key = 3, value = 2030-04-15T00:00:00.000000Z -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), +(2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), +(3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(4, NULL); + )", R"( +key = 3, value = 2030-04-15T00:00:00.000000Z +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldNotErase) { - ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(2, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )", R"( -key = 1, value = 2030-04-15T00:00:00.000000Z -key = 2, value = 2030-04-15T00:00:00.000000Z -key = 3, value = 2030-04-15T00:00:00.000000Z + ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(2, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )", R"( +key = 1, value = 2030-04-15T00:00:00.000000Z +key = 2, value = 2030-04-15T00:00:00.000000Z +key = 3, value = 2030-04-15T00:00:00.000000Z )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnUint32) { - ConditionalEraseShouldSuccess("Uint32", TUnit::SECONDS, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, 0), -(2, 636249600), -(3, 1902441600), -(4, NULL); - )", R"( -key = 3, value = 1902441600 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Uint32", TUnit::SECONDS, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, 0), +(2, 636249600), +(3, 1902441600), +(4, NULL); + )", R"( +key = 3, value = 1902441600 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnUint64Seconds) { - ConditionalEraseShouldSuccess("Uint64", TUnit::SECONDS, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, 0), -(2, 636249600), -(3, 1902441600), -(4, NULL); - )", R"( -key = 3, value = 1902441600 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Uint64", TUnit::SECONDS, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, 0), +(2, 636249600), +(3, 1902441600), +(4, NULL); + )", R"( +key = 3, value = 1902441600 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnUint64MilliSeconds) { - ConditionalEraseShouldSuccess("Uint64", TUnit::MILLISECONDS, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, 0), -(2, 636249600000), -(3, 1902441600000), -(4, NULL); - )", R"( -key = 3, value = 1902441600000 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Uint64", TUnit::MILLISECONDS, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, 0), +(2, 636249600000), +(3, 1902441600000), +(4, NULL); + )", R"( +key = 3, value = 1902441600000 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnUint64MicroSeconds) { - ConditionalEraseShouldSuccess("Uint64", TUnit::MICROSECONDS, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, 0), -(2, 636249600000000), -(3, 1902441600000000), -(4, NULL); - )", R"( -key = 3, value = 1902441600000000 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Uint64", TUnit::MICROSECONDS, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, 0), +(2, 636249600000000), +(3, 1902441600000000), +(4, NULL); + )", R"( +key = 3, value = 1902441600000000 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnUint64NanoSeconds) { - ConditionalEraseShouldSuccess("Uint64", TUnit::NANOSECONDS, R"( -UPSERT INTO [/Root/table-1] (key, value) VALUES -(1, 0), -(2, 636249600000000000), -(3, 1902441600000000000), -(4, NULL); - )", R"( -key = 3, value = 1902441600000000000 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("Uint64", TUnit::NANOSECONDS, R"( +UPSERT INTO [/Root/table-1] (key, value) VALUES +(1, 0), +(2, 636249600000000000), +(3, 1902441600000000000), +(4, NULL); + )", R"( +key = 3, value = 1902441600000000000 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnDyNumberSeconds) { - ConditionalEraseShouldSuccess("DyNumber", TUnit::SECONDS, R"( ---!syntax_v1 -UPSERT INTO `/Root/table-1` (key, value) VALUES -(1, CAST("0" As DyNumber)), -(2, CAST("636249600" As DyNumber)), -(3, CAST("636249600.123" As DyNumber)), -(4, CAST("-1902441600" As DyNumber)), -(5, CAST("1902441600" As DyNumber)), -(6, CAST("636249600000" As DyNumber)), -(7, NULL); - )", R"( -key = 5, value = .190244160e10 -key = 6, value = .63624960000e12 -key = 7, value = (empty maybe) + ConditionalEraseShouldSuccess("DyNumber", TUnit::SECONDS, R"( +--!syntax_v1 +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("0" As DyNumber)), +(2, CAST("636249600" As DyNumber)), +(3, CAST("636249600.123" As DyNumber)), +(4, CAST("-1902441600" As DyNumber)), +(5, CAST("1902441600" As DyNumber)), +(6, CAST("636249600000" As DyNumber)), +(7, NULL); + )", R"( +key = 5, value = .190244160e10 +key = 6, value = .63624960000e12 +key = 7, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnDyNumberMilliSeconds) { - ConditionalEraseShouldSuccess("DyNumber", TUnit::MILLISECONDS, R"( ---!syntax_v1 -UPSERT INTO `/Root/table-1` (key, value) VALUES -(1, CAST("0" As DyNumber)), -(2, CAST("636249600000" As DyNumber)), -(3, CAST("1902441600000" As DyNumber)), -(4, CAST("636249600000000" As DyNumber)), -(5, NULL); - )", R"( -key = 3, value = .1902441600000e13 -key = 4, value = .636249600000000e15 -key = 5, value = (empty maybe) + ConditionalEraseShouldSuccess("DyNumber", TUnit::MILLISECONDS, R"( +--!syntax_v1 +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("0" As DyNumber)), +(2, CAST("636249600000" As DyNumber)), +(3, CAST("1902441600000" As DyNumber)), +(4, CAST("636249600000000" As DyNumber)), +(5, NULL); + )", R"( +key = 3, value = .1902441600000e13 +key = 4, value = .636249600000000e15 +key = 5, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnDyNumberMicroSeconds) { - ConditionalEraseShouldSuccess("DyNumber", TUnit::MICROSECONDS, R"( ---!syntax_v1 -UPSERT INTO `/Root/table-1` (key, value) VALUES -(1, CAST("-9.9999999999999999999999999999999999999E+125" As DyNumber)), -(2, CAST("-1E-130" As DyNumber)), -(3, CAST("0" As DyNumber)), -(4, CAST("1E-130" As DyNumber)), -(5, CAST("636249600000000" As DyNumber)), -(6, CAST("1902441600000000" As DyNumber)), -(7, CAST("9.9999999999999999999999999999999999999E+125" As DyNumber)), -(8, NULL); - )", R"( -key = 6, value = .190244160000000e16 -key = 7, value = .99999999999999999999999999999999999999e126 -key = 8, value = (empty maybe) + ConditionalEraseShouldSuccess("DyNumber", TUnit::MICROSECONDS, R"( +--!syntax_v1 +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("-9.9999999999999999999999999999999999999E+125" As DyNumber)), +(2, CAST("-1E-130" As DyNumber)), +(3, CAST("0" As DyNumber)), +(4, CAST("1E-130" As DyNumber)), +(5, CAST("636249600000000" As DyNumber)), +(6, CAST("1902441600000000" As DyNumber)), +(7, CAST("9.9999999999999999999999999999999999999E+125" As DyNumber)), +(8, NULL); + )", R"( +key = 6, value = .190244160000000e16 +key = 7, value = .99999999999999999999999999999999999999e126 +key = 8, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldEraseOnDyNumberNanoSeconds) { - ConditionalEraseShouldSuccess("DyNumber", TUnit::NANOSECONDS, R"( ---!syntax_v1 -UPSERT INTO `/Root/table-1` (key, value) VALUES -(1, CAST("0" As DyNumber)), -(2, CAST("636249600000000000" As DyNumber)), -(3, CAST("1902441600000000000" As DyNumber)), -(4, NULL); - )", R"( -key = 3, value = .1902441600000000000e19 -key = 4, value = (empty maybe) + ConditionalEraseShouldSuccess("DyNumber", TUnit::NANOSECONDS, R"( +--!syntax_v1 +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("0" As DyNumber)), +(2, CAST("636249600000000000" As DyNumber)), +(3, CAST("1902441600000000000" As DyNumber)), +(4, NULL); + )", R"( +key = 3, value = .1902441600000000000e19 +key = 4, value = (empty maybe) )", WithMvcc); - } - + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldFailOnVariousErrors) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(WithMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - int tableNum = 1; - for (TStringBuf ct : {"Date", "Datetime", "Timestamp", "Uint32", "Uint64", "DyNumber", "String"}) { - const auto tableName = Sprintf("table-%i", tableNum++); - const auto tablePath = Sprintf("/Root/%s", tableName.data()); - + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + int tableNum = 1; + for (TStringBuf ct : {"Date", "Datetime", "Timestamp", "Uint32", "Uint64", "DyNumber", "String"}) { + const auto tableName = Sprintf("table-%i", tableNum++); + const auto tablePath = Sprintf("/Root/%s", tableName.data()); + CreateTable(server, sender, "/Root", tableName, TString(ct)); - auto tableId = ResolveTableId(server, sender, tablePath); - - ConditionalEraseRows(server, sender, tablePath, TTableId(), 2, 0, TUnit::AUTO, {}, {}, - TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown table id"); - - ConditionalEraseRows(server, sender, tablePath, tableId, 3, 0, TUnit::AUTO, {}, {}, - TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown column id"); - - if (ct == "Date" || ct == "Datetime" || ct == "Timestamp") { - for (auto unit : {TUnit::SECONDS, TUnit::MILLISECONDS, TUnit::MICROSECONDS}) { - ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, unit, {}, {}, - TEvResponse::ProtoRecordType::BAD_REQUEST, "Unit cannot be specified for date type column"); - } - } else if (ct == "Uint32" || ct == "Uint64" || ct == "DyNumber") { - ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, TUnit::AUTO, {}, {}, - TEvResponse::ProtoRecordType::BAD_REQUEST, "Unit should be specified for integral type column"); - } else { - ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, TUnit::AUTO, {}, {}, - TEvResponse::ProtoRecordType::BAD_REQUEST, "Unsupported type"); - } - } - } - + auto tableId = ResolveTableId(server, sender, tablePath); + + ConditionalEraseRows(server, sender, tablePath, TTableId(), 2, 0, TUnit::AUTO, {}, {}, + TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown table id"); + + ConditionalEraseRows(server, sender, tablePath, tableId, 3, 0, TUnit::AUTO, {}, {}, + TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown column id"); + + if (ct == "Date" || ct == "Datetime" || ct == "Timestamp") { + for (auto unit : {TUnit::SECONDS, TUnit::MILLISECONDS, TUnit::MICROSECONDS}) { + ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, unit, {}, {}, + TEvResponse::ProtoRecordType::BAD_REQUEST, "Unit cannot be specified for date type column"); + } + } else if (ct == "Uint32" || ct == "Uint64" || ct == "DyNumber") { + ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, TUnit::AUTO, {}, {}, + TEvResponse::ProtoRecordType::BAD_REQUEST, "Unit should be specified for integral type column"); + } else { + ConditionalEraseRows(server, sender, tablePath, tableId, 2, 0, TUnit::AUTO, {}, {}, + TEvResponse::ProtoRecordType::BAD_REQUEST, "Unsupported type"); + } + } + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldBreakLocks) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(WithMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + CreateTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, value) VALUES - (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - TString sessionId = CreateSession(runtime, sender); - - TString txId; - { - auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, R"( - SELECT value FROM [/Root/table-1] WHERE key = 1 - )")); - - auto& response = ev->Get()->Record.GetRef(); - UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1); - - txId = response.GetResponse().GetTxMeta().id(); - } - - { - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue()); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - } - - { - auto ev = ExecRequest(runtime, sender, MakeCommitRequest(sessionId, txId, R"( - UPSERT INTO [/Root/table-1] (key, value) VALUES - (4, CAST("2031-04-15T00:00:00.000000Z" AS Timestamp)); - )")); - - auto& response = ev->Get()->Record.GetRef(); - UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, value) VALUES + (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + TString sessionId = CreateSession(runtime, sender); + + TString txId; + { + auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, R"( + SELECT value FROM [/Root/table-1] WHERE key = 1 + )")); + + auto& response = ev->Get()->Record.GetRef(); + UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1); + + txId = response.GetResponse().GetTxMeta().id(); + } + + { + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue()); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + } + + { + auto ev = ExecRequest(runtime, sender, MakeCommitRequest(sessionId, txId, R"( + UPSERT INTO [/Root/table-1] (key, value) VALUES + (4, CAST("2031-04-15T00:00:00.000000Z" AS Timestamp)); + )")); + + auto& response = ev->Get()->Record.GetRef(); + UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED); NYql::TIssues issues; IssuesFromMessage(response.GetResponse().GetQueryIssues(), issues); UNIT_ASSERT(NKqp::HasIssue(issues, NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED)); - } - } - + } + } + Y_UNIT_TEST_WITH_MVCC(ConditionalEraseRowsShouldNotEraseModifiedRows) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings .SetEnableMvcc(WithMvcc) - .SetDomainName("Root") + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + CreateTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, value) VALUES - (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - THolder<IEventHandle> delayed; - auto prevObserver = server->GetRuntime()->SetObserverFunc([&delayed](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvDataShard::TEvEraseRowsRequest::EventType: - delayed.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue()); - - if (!delayed) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return !!delayed; - }); - server->GetRuntime()->DispatchEvents(opts); - } - - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, value) VALUES - (3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - server->GetRuntime()->SetObserverFunc(prevObserver); - server->GetRuntime()->Send(delayed.Release(), 0, true); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - - auto content = ReadShardedTable(server, "/Root/table-1"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, value = 2030-04-15T00:00:00.000000Z"); - } -} - -Y_UNIT_TEST_SUITE(DistributedEraseTests) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TResolve = NSchemeCache::TSchemeCacheRequest; - - void FillPath(TNavigate::TEntry& entry, const TString& path) { - entry.Path = SplitPath(path); - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - } - - void FillPath(TNavigate::TEntry& entry, const TTableId& tableId) { - entry.TableId = tableId; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - } - - template <typename TPath> - THolder<TNavigate> Navigate(TServer::TPtr server, const TActorId& sender, const TPath& path, TNavigate::EOp op) { - using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; - - auto& runtime = *server->GetRuntime(); - - auto request = MakeHolder<TNavigate>(); - auto& entry = request->ResultSet.emplace_back(); - FillPath(entry, path); - entry.Operation = op; - entry.ShowPrivatePath = true; - runtime.Send(new IEventHandle(MakeSchemeCacheID(), sender, new TEvRequest(request.Release()))); - - auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()); - - auto* response = ev->Get()->Request.Release(); - UNIT_ASSERT(response); - UNIT_ASSERT(response->ErrorCount == 0); - UNIT_ASSERT_VALUES_EQUAL(response->ResultSet.size(), 1); - + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, value) VALUES + (1, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + THolder<IEventHandle> delayed; + auto prevObserver = server->GetRuntime()->SetObserverFunc([&delayed](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvDataShard::TEvEraseRowsRequest::EventType: + delayed.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 2, TInstant::Now().GetValue()); + + if (!delayed) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return !!delayed; + }); + server->GetRuntime()->DispatchEvents(opts); + } + + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, value) VALUES + (3, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + server->GetRuntime()->SetObserverFunc(prevObserver); + server->GetRuntime()->Send(delayed.Release(), 0, true); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + + auto content = ReadShardedTable(server, "/Root/table-1"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, value = 2030-04-15T00:00:00.000000Z"); + } +} + +Y_UNIT_TEST_SUITE(DistributedEraseTests) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TResolve = NSchemeCache::TSchemeCacheRequest; + + void FillPath(TNavigate::TEntry& entry, const TString& path) { + entry.Path = SplitPath(path); + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + } + + void FillPath(TNavigate::TEntry& entry, const TTableId& tableId) { + entry.TableId = tableId; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + } + + template <typename TPath> + THolder<TNavigate> Navigate(TServer::TPtr server, const TActorId& sender, const TPath& path, TNavigate::EOp op) { + using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; + + auto& runtime = *server->GetRuntime(); + + auto request = MakeHolder<TNavigate>(); + auto& entry = request->ResultSet.emplace_back(); + FillPath(entry, path); + entry.Operation = op; + entry.ShowPrivatePath = true; + runtime.Send(new IEventHandle(MakeSchemeCacheID(), sender, new TEvRequest(request.Release()))); + + auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()); + + auto* response = ev->Get()->Request.Release(); + UNIT_ASSERT(response); + UNIT_ASSERT(response->ErrorCount == 0); + UNIT_ASSERT_VALUES_EQUAL(response->ResultSet.size(), 1); + return THolder(response); - } - + } + NDataShard::TIndexes GetIndexes(TServer::TPtr server, const TActorId& sender, const TString& path) { - auto mainTable = Navigate(server, sender, path, TNavigate::OpTable); - const auto& mainEntry = mainTable->ResultSet.at(0); - - TVector<ui32> keyOrder; - THashMap<TString, ui32> columnNameToId; - for (const auto& [id, column] : mainEntry.Columns) { - if (column.KeyOrder >= 0) { - if (keyOrder.size() < static_cast<ui32>(column.KeyOrder + 1)) { - keyOrder.resize(column.KeyOrder + 1); - } - keyOrder[column.KeyOrder] = id; - } - - UNIT_ASSERT(columnNameToId.emplace(column.Name, id).second); - } - + auto mainTable = Navigate(server, sender, path, TNavigate::OpTable); + const auto& mainEntry = mainTable->ResultSet.at(0); + + TVector<ui32> keyOrder; + THashMap<TString, ui32> columnNameToId; + for (const auto& [id, column] : mainEntry.Columns) { + if (column.KeyOrder >= 0) { + if (keyOrder.size() < static_cast<ui32>(column.KeyOrder + 1)) { + keyOrder.resize(column.KeyOrder + 1); + } + keyOrder[column.KeyOrder] = id; + } + + UNIT_ASSERT(columnNameToId.emplace(column.Name, id).second); + } + NDataShard::TIndexes indexes; - for (const auto& index : mainEntry.Indexes) { - if (index.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync) { - continue; - } - - THashSet<TString> seen; - + for (const auto& index : mainEntry.Indexes) { + if (index.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync) { + continue; + } + + THashSet<TString> seen; + NDataShard::TKeyMap keyMap; - ui32 i = 0; - for (; i < index.KeyColumnNamesSize(); ++i) { - const TString& name = index.GetKeyColumnNames(i); - seen.insert(name); - - UNIT_ASSERT(columnNameToId.contains(name)); - keyMap.emplace_back(std::make_pair(i + 1, columnNameToId.at(name))); - } - - for (const ui32 id : keyOrder) { - UNIT_ASSERT(mainEntry.Columns.contains(id)); - if (seen.contains(mainEntry.Columns.at(id).Name)) { - continue; - } - - keyMap.emplace_back(std::make_pair(++i, id)); - } - - const auto tableId = TTableId(index.GetPathOwnerId(), index.GetLocalPathId(), index.GetSchemaVersion()); - auto indexTable = Navigate(server, sender, tableId, TNavigate::OpList); - const auto& indexEntry = indexTable->ResultSet.at(0); - - UNIT_ASSERT(indexEntry.ListNodeEntry); - UNIT_ASSERT_VALUES_EQUAL(indexEntry.ListNodeEntry->Children.size(), 1); - const auto& child = indexEntry.ListNodeEntry->Children.at(0); - - UNIT_ASSERT(indexes.emplace(TTableId(child.PathId, child.SchemaVersion), std::move(keyMap)).second); - } - - return indexes; - } - - TVector<THolder<IEventHandle>> ConditionalEraseRowsDelayedPlan( - TServer::TPtr server, const TActorId& sender, const TString& path, + ui32 i = 0; + for (; i < index.KeyColumnNamesSize(); ++i) { + const TString& name = index.GetKeyColumnNames(i); + seen.insert(name); + + UNIT_ASSERT(columnNameToId.contains(name)); + keyMap.emplace_back(std::make_pair(i + 1, columnNameToId.at(name))); + } + + for (const ui32 id : keyOrder) { + UNIT_ASSERT(mainEntry.Columns.contains(id)); + if (seen.contains(mainEntry.Columns.at(id).Name)) { + continue; + } + + keyMap.emplace_back(std::make_pair(++i, id)); + } + + const auto tableId = TTableId(index.GetPathOwnerId(), index.GetLocalPathId(), index.GetSchemaVersion()); + auto indexTable = Navigate(server, sender, tableId, TNavigate::OpList); + const auto& indexEntry = indexTable->ResultSet.at(0); + + UNIT_ASSERT(indexEntry.ListNodeEntry); + UNIT_ASSERT_VALUES_EQUAL(indexEntry.ListNodeEntry->Children.size(), 1); + const auto& child = indexEntry.ListNodeEntry->Children.at(0); + + UNIT_ASSERT(indexes.emplace(TTableId(child.PathId, child.SchemaVersion), std::move(keyMap)).second); + } + + return indexes; + } + + TVector<THolder<IEventHandle>> ConditionalEraseRowsDelayedPlan( + TServer::TPtr server, const TActorId& sender, const TString& path, const TTableId& tableId, ui32 columnId, ui64 threshold, const NDataShard::TIndexes& indexes) { - - TVector<THolder<IEventHandle>> delayed; - - auto& runtime = *server->GetRuntime(); - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvTxProxy::TEvProposeTransaction::EventType: - delayed.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - ConditionalEraseRows(server, sender, path, tableId, columnId, threshold, TUnit::AUTO, indexes); - - if (delayed.size() < 1) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return delayed.size() >= 1; - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - return delayed; - } - - TVector<THolder<IEventHandle>> ConditionalEraseRowsDelayedResolve( - TServer::TPtr server, const TActorId& sender, const TString& path, + + TVector<THolder<IEventHandle>> delayed; + + auto& runtime = *server->GetRuntime(); + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvTxProxy::TEvProposeTransaction::EventType: + delayed.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + ConditionalEraseRows(server, sender, path, tableId, columnId, threshold, TUnit::AUTO, indexes); + + if (delayed.size() < 1) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return delayed.size() >= 1; + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + return delayed; + } + + TVector<THolder<IEventHandle>> ConditionalEraseRowsDelayedResolve( + TServer::TPtr server, const TActorId& sender, const TString& path, const TTableId& tableId, ui32 columnId, ui64 threshold, const NDataShard::TIndexes& indexes) { - - TVector<THolder<IEventHandle>> delayed; - TActorId eraser; - - auto& runtime = *server->GetRuntime(); - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvDataShard::TEvEraseRowsRequest::EventType: - delayed.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - case TEvTxProxySchemeCache::TEvNavigateKeySetResult::EventType: - if (ev->Recipient == eraser) { - delayed.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } - case TEvTxUserProxy::TEvAllocateTxId::EventType: - eraser = ev->Sender; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - ConditionalEraseRows(server, sender, path, tableId, columnId, threshold, TUnit::AUTO, indexes); - - if (!eraser) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&eraser](IEventHandle&) -> bool { - return !!eraser; - }); - runtime.DispatchEvents(opts); - } - - if (delayed.size() < 2) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return delayed.size() >= 2; - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - return delayed; - } - - void ConditionalEraseShouldSuccess( - const TString& ttlColType, EUnit unit, - const TString& toUpload, const THashMap<TString, TString>& afterErase, - const THashMap<TString, ui32>& shardsConfig = {}) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + TVector<THolder<IEventHandle>> delayed; + TActorId eraser; + + auto& runtime = *server->GetRuntime(); + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvDataShard::TEvEraseRowsRequest::EventType: + delayed.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + case TEvTxProxySchemeCache::TEvNavigateKeySetResult::EventType: + if (ev->Recipient == eraser) { + delayed.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + case TEvTxUserProxy::TEvAllocateTxId::EventType: + eraser = ev->Sender; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + ConditionalEraseRows(server, sender, path, tableId, columnId, threshold, TUnit::AUTO, indexes); + + if (!eraser) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&eraser](IEventHandle&) -> bool { + return !!eraser; + }); + runtime.DispatchEvents(opts); + } + + if (delayed.size() < 2) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return delayed.size() >= 2; + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + return delayed; + } + + void ConditionalEraseShouldSuccess( + const TString& ttlColType, EUnit unit, + const TString& toUpload, const THashMap<TString, TString>& afterErase, + const THashMap<TString, ui32>& shardsConfig = {}) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1", ttlColType); - ExecSQL(server, sender, toUpload); - - if (shardsConfig) { - SimulateSleep(server, TDuration::Seconds(1)); - SetSplitMergePartCountLimit(&runtime, -1); - } - - for (const auto& [path, splitKey] : shardsConfig) { - auto tabletIds = GetTableShards(server, sender, path); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - const ui64 txId = AsyncSplitTable(server, sender, path, tabletIds[0], splitKey); - WaitTxNotification(server, sender, txId); - } - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 4, TInstant::Now().GetValue(), unit, indexes); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - - for (const auto& [path, expectedContent] : afterErase) { - auto content = ReadShardedTable(server, path); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1", ttlColType); + ExecSQL(server, sender, toUpload); + + if (shardsConfig) { + SimulateSleep(server, TDuration::Seconds(1)); + SetSplitMergePartCountLimit(&runtime, -1); + } + + for (const auto& [path, splitKey] : shardsConfig) { + auto tabletIds = GetTableShards(server, sender, path); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + const ui64 txId = AsyncSplitTable(server, sender, path, tabletIds[0], splitKey); + WaitTxNotification(server, sender, txId); + } + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 4, TInstant::Now().GetValue(), unit, indexes); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + + for (const auto& [path, expectedContent] : afterErase) { + auto content = ReadShardedTable(server, path); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), Strip(expectedContent)); - } - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldErase) { - ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( -UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES -(1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), -(2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), -(3, 30, 200, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(4, 40, 100, NULL); - )", {{ -"/Root/table-1", R"( -key = 3, skey = 30, tkey = 200, value = 2030-04-15T00:00:00.000000Z -key = 4, skey = 40, tkey = 100, value = (empty maybe) - )"}, { -"/Root/table-1/by_skey/indexImplTable", R"( -skey = 30, key = 3 -skey = 40, key = 4 - )"}, { -"/Root/table-1/by_tkey/indexImplTable", R"( -tkey = 100, key = 4 -tkey = 200, key = 3 - )"}}); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldEraseOnUint32) { - ConditionalEraseShouldSuccess("Uint32", TUnit::SECONDS, R"( -UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES -(1, 10, 400, 0), -(2, 20, 300, 636249600), -(3, 30, 200, 1902441600), -(4, 40, 100, NULL); - )", {{ -"/Root/table-1", R"( -key = 3, skey = 30, tkey = 200, value = 1902441600 -key = 4, skey = 40, tkey = 100, value = (empty maybe) - )"}, { -"/Root/table-1/by_skey/indexImplTable", R"( -skey = 30, key = 3 -skey = 40, key = 4 - )"}, { -"/Root/table-1/by_tkey/indexImplTable", R"( -tkey = 100, key = 4 -tkey = 200, key = 3 - )"}}); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldNotErase) { - ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( -UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES -(1, 10, 300, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(2, 20, 200, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), -(3, 30, 100, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )", {{ -"/Root/table-1", R"( -key = 1, skey = 10, tkey = 300, value = 2030-04-15T00:00:00.000000Z -key = 2, skey = 20, tkey = 200, value = 2030-04-15T00:00:00.000000Z -key = 3, skey = 30, tkey = 100, value = 2030-04-15T00:00:00.000000Z - )"}, { -"/Root/table-1/by_skey/indexImplTable", R"( -skey = 10, key = 1 -skey = 20, key = 2 -skey = 30, key = 3 - )"}, { -"/Root/table-1/by_tkey/indexImplTable", R"( -tkey = 100, key = 3 -tkey = 200, key = 2 -tkey = 300, key = 1 - )"}}); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldSuccessOnShardedIndex) { - ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( -UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES -(1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), -(2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), -(3, 30, 200, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)), -(4, 40, 100, NULL); - )", {{ -"/Root/table-1", R"( -key = 4, skey = 40, tkey = 100, value = (empty maybe) - )"}, { -"/Root/table-1/by_skey/indexImplTable", R"( -skey = 40, key = 4 - )"}, { -"/Root/table-1/by_tkey/indexImplTable", R"( -tkey = 100, key = 4 - )"}}, {{ -"/Root/table-1/by_skey/indexImplTable", 20 - }, { -"/Root/table-1/by_tkey/indexImplTable", 300 - }}); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldNotEraseModifiedRows) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + } + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldErase) { + ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( +UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES +(1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), +(2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), +(3, 30, 200, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(4, 40, 100, NULL); + )", {{ +"/Root/table-1", R"( +key = 3, skey = 30, tkey = 200, value = 2030-04-15T00:00:00.000000Z +key = 4, skey = 40, tkey = 100, value = (empty maybe) + )"}, { +"/Root/table-1/by_skey/indexImplTable", R"( +skey = 30, key = 3 +skey = 40, key = 4 + )"}, { +"/Root/table-1/by_tkey/indexImplTable", R"( +tkey = 100, key = 4 +tkey = 200, key = 3 + )"}}); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldEraseOnUint32) { + ConditionalEraseShouldSuccess("Uint32", TUnit::SECONDS, R"( +UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES +(1, 10, 400, 0), +(2, 20, 300, 636249600), +(3, 30, 200, 1902441600), +(4, 40, 100, NULL); + )", {{ +"/Root/table-1", R"( +key = 3, skey = 30, tkey = 200, value = 1902441600 +key = 4, skey = 40, tkey = 100, value = (empty maybe) + )"}, { +"/Root/table-1/by_skey/indexImplTable", R"( +skey = 30, key = 3 +skey = 40, key = 4 + )"}, { +"/Root/table-1/by_tkey/indexImplTable", R"( +tkey = 100, key = 4 +tkey = 200, key = 3 + )"}}); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldNotErase) { + ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( +UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES +(1, 10, 300, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(2, 20, 200, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)), +(3, 30, 100, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )", {{ +"/Root/table-1", R"( +key = 1, skey = 10, tkey = 300, value = 2030-04-15T00:00:00.000000Z +key = 2, skey = 20, tkey = 200, value = 2030-04-15T00:00:00.000000Z +key = 3, skey = 30, tkey = 100, value = 2030-04-15T00:00:00.000000Z + )"}, { +"/Root/table-1/by_skey/indexImplTable", R"( +skey = 10, key = 1 +skey = 20, key = 2 +skey = 30, key = 3 + )"}, { +"/Root/table-1/by_tkey/indexImplTable", R"( +tkey = 100, key = 3 +tkey = 200, key = 2 +tkey = 300, key = 1 + )"}}); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldSuccessOnShardedIndex) { + ConditionalEraseShouldSuccess("Timestamp", TUnit::AUTO, R"( +UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES +(1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), +(2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), +(3, 30, 200, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)), +(4, 40, 100, NULL); + )", {{ +"/Root/table-1", R"( +key = 4, skey = 40, tkey = 100, value = (empty maybe) + )"}, { +"/Root/table-1/by_skey/indexImplTable", R"( +skey = 40, key = 4 + )"}, { +"/Root/table-1/by_tkey/indexImplTable", R"( +tkey = 100, key = 4 + )"}}, {{ +"/Root/table-1/by_skey/indexImplTable", 20 + }, { +"/Root/table-1/by_tkey/indexImplTable", 300 + }}); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldNotEraseModifiedRows) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - // case 1: modify ttl column - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (3, 90, 900, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - { - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - } - { - auto content = ReadShardedTable(server, "/Root/table-1"); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + // case 1: modify ttl column + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (3, 90, 900, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + { + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + } + { + auto content = ReadShardedTable(server, "/Root/table-1"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, skey = 90, tkey = 900, value = 2030-04-15T00:00:00.000000Z"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "skey = 90, key = 3"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "tkey = 900, key = 3"); - } - - // restore previous value in ttl column - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (3, 90, 900, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - // case 2: modify index column - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - { - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - } - { - auto content = ReadShardedTable(server, "/Root/table-1"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, skey = 30, tkey = 100, value = 2020-04-15T00:00:00.000000Z"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "skey = 30, key = 3"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "tkey = 100, key = 3"); - } - - // after one more run, all records should be deleted - ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes); - { - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - } - { - auto content = ReadShardedTable(server, "/Root/table-1"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); - } - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldNotFailOnMissingRows) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + } + + // restore previous value in ttl column + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (3, 90, 900, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + // case 2: modify index column + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + { + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + } + { + auto content = ReadShardedTable(server, "/Root/table-1"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, skey = 30, tkey = 100, value = 2020-04-15T00:00:00.000000Z"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "skey = 30, key = 3"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "tkey = 100, key = 3"); + } + + // after one more run, all records should be deleted + ConditionalEraseRows(server, sender, "/Root/table-1", tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes); + { + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + } + { + auto content = ReadShardedTable(server, "/Root/table-1"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); + } + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldNotFailOnMissingRows) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 200, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)), - (4, 40, 100, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - ExecSQL(server, sender, "DELETE FROM [/Root/table-1] WHERE key < 3;"); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - - { - auto content = ReadShardedTable(server, "/Root/table-1"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 4, skey = 40, tkey = 100, value = 2030-04-15T00:00:00.000000Z"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "skey = 40, key = 4"); - } - { - auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "tkey = 100, key = 4"); - } - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnVariousErrors) { - using TEvRequest = TEvDataShard::TEvEraseRowsRequest; - using TEvResponse = TEvDataShard::TEvEraseRowsResponse; - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySetResult; - using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySetResult; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (1, 10, 400, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 300, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 200, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)), + (4, 40, 100, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + ExecSQL(server, sender, "DELETE FROM [/Root/table-1] WHERE key < 3;"); + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + + { + auto content = ReadShardedTable(server, "/Root/table-1"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 4, skey = 40, tkey = 100, value = 2030-04-15T00:00:00.000000Z"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_skey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "skey = 40, key = 4"); + } + { + auto content = ReadShardedTable(server, "/Root/table-1/by_tkey/indexImplTable"); + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "tkey = 100, key = 4"); + } + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnVariousErrors) { + using TEvRequest = TEvDataShard::TEvEraseRowsRequest; + using TEvResponse = TEvDataShard::TEvEraseRowsResponse; + using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySetResult; + using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySetResult; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - - using TSimpleEventObserver = std::function<void(TAutoPtr<IEventHandle>&)>; - auto check = [&](TEvResponse::ProtoRecordType::EStatus status, const TString& error, TSimpleEventObserver observer) { - auto prevObserver = runtime.SetObserverFunc([observer](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - observer(ev); - return TTestActorRuntime::EEventAction::PROCESS; - }); - + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + + using TSimpleEventObserver = std::function<void(TAutoPtr<IEventHandle>&)>; + auto check = [&](TEvResponse::ProtoRecordType::EStatus status, const TString& error, TSimpleEventObserver observer) { + auto prevObserver = runtime.SetObserverFunc([observer](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + observer(ev); + return TTestActorRuntime::EEventAction::PROCESS; + }); + const auto eraser = runtime.Register(NDataShard::CreateDistributedEraser(sender, tableId, indexes)); - auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); - UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); - runtime.Send(new IEventHandle(eraser, sender, new TEvents::TEvPoisonPill())); - - runtime.SetObserverFunc(prevObserver); - }; - - /// resolve tables - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty result", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request.Reset(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Entries count mismatch", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.clear(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).Status = TNavigate::EStatus::Unknown; - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Main table's path id mismatch", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).TableId = TTableId(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Main table's schema version mismatch", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).TableId.SchemaVersion = 0; - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Indexes count mismatch", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).Indexes.clear(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Invalid index state", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); + UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); + runtime.Send(new IEventHandle(eraser, sender, new TEvents::TEvPoisonPill())); + + runtime.SetObserverFunc(prevObserver); + }; + + /// resolve tables + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty result", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request.Reset(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Entries count mismatch", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.clear(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).Status = TNavigate::EStatus::Unknown; + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Main table's path id mismatch", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).TableId = TTableId(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Main table's schema version mismatch", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).TableId.SchemaVersion = 0; + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Indexes count mismatch", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).Indexes.clear(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Invalid index state", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { ev->Get<TEvNavigate>()->Request->ResultSet.at(0).Indexes.at(0).SetState(NKikimrSchemeOp::EIndexStateInvalid); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(1).Status = TNavigate::EStatus::Unknown; - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Unknown index", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(1).TableId = TTableId(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Duplicate index", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - auto& resultSet = ev->Get<TEvNavigate>()->Request->ResultSet; - resultSet.at(1).TableId = resultSet.at(2).TableId; - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty domain info", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).DomainInfo.Drop(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed locality check", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvNavigate::EventType) { - ev->Get<TEvNavigate>()->Request->ResultSet.at(0).DomainInfo = new NSchemeCache::TDomainInfo(TPathId(), TPathId()); - } - }); - - /// resolve keys - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty result", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvResolve::EventType) { - ev->Get<TEvResolve>()->Request.Reset(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Entries count mismatch", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvResolve::EventType) { - ev->Get<TEvResolve>()->Request->ResultSet.clear(); - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvResolve::EventType) { - ev->Get<TEvResolve>()->Request->ResultSet.at(0).Status = TResolve::EStatus::Unknown; - } - }); - - check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty partitions list", [](TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvResolve::EventType) { - ev->Get<TEvResolve>()->Request->ResultSet.at(0).KeyDescription->Partitions.clear(); - } - }); - - /// process request - - using TRequestMaker = std::function<IEventBase*(void)>; - auto badRequest = [&](TEvResponse::ProtoRecordType::EStatus status, const TString& error, TRequestMaker maker) { + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(1).Status = TNavigate::EStatus::Unknown; + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Unknown index", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(1).TableId = TTableId(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Duplicate index", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + auto& resultSet = ev->Get<TEvNavigate>()->Request->ResultSet; + resultSet.at(1).TableId = resultSet.at(2).TableId; + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty domain info", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).DomainInfo.Drop(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed locality check", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvNavigate::EventType) { + ev->Get<TEvNavigate>()->Request->ResultSet.at(0).DomainInfo = new NSchemeCache::TDomainInfo(TPathId(), TPathId()); + } + }); + + /// resolve keys + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty result", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvResolve::EventType) { + ev->Get<TEvResolve>()->Request.Reset(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Entries count mismatch", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvResolve::EventType) { + ev->Get<TEvResolve>()->Request->ResultSet.clear(); + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Failed to resolve table", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvResolve::EventType) { + ev->Get<TEvResolve>()->Request->ResultSet.at(0).Status = TResolve::EStatus::Unknown; + } + }); + + check(TEvResponse::ProtoRecordType::SCHEME_ERROR, "Empty partitions list", [](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvResolve::EventType) { + ev->Get<TEvResolve>()->Request->ResultSet.at(0).KeyDescription->Partitions.clear(); + } + }); + + /// process request + + using TRequestMaker = std::function<IEventBase*(void)>; + auto badRequest = [&](TEvResponse::ProtoRecordType::EStatus status, const TString& error, TRequestMaker maker) { const auto eraser = runtime.Register(NDataShard::CreateDistributedEraser(sender, tableId, indexes)); - - runtime.Send(new IEventHandle(eraser, sender, maker()), 0, true); - auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); - UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); - - runtime.Send(new IEventHandle(eraser, sender, new TEvents::TEvPoisonPill())); - }; - - badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown condition", []() -> IEventBase* { - return new TEvRequest(); - }); - - badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Cannot parse key", []() -> IEventBase* { - auto request = MakeHolder<TEvRequest>(); - request->Record.MutableExpiration(); - request->Record.AddKeyColumns("trash"); - return request.Release(); - }); - - badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Key column is absent", []() -> IEventBase* { - auto request = MakeHolder<TEvRequest>(); - request->Record.MutableExpiration(); - for (const auto& key : SerializeKeys({1, 2})) { - request->Record.AddKeyColumns(key); - } - return request.Release(); - }); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnSplit) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + runtime.Send(new IEventHandle(eraser, sender, maker()), 0, true); + auto ev = runtime.GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), status); + UNIT_ASSERT_STRING_CONTAINS(ev->Get()->Record.GetErrorDescription(), error); + + runtime.Send(new IEventHandle(eraser, sender, new TEvents::TEvPoisonPill())); + }; + + badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Unknown condition", []() -> IEventBase* { + return new TEvRequest(); + }); + + badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Cannot parse key", []() -> IEventBase* { + auto request = MakeHolder<TEvRequest>(); + request->Record.MutableExpiration(); + request->Record.AddKeyColumns("trash"); + return request.Release(); + }); + + badRequest(TEvResponse::ProtoRecordType::BAD_REQUEST, "Key column is absent", []() -> IEventBase* { + auto request = MakeHolder<TEvRequest>(); + request->Record.MutableExpiration(); + for (const auto& key : SerializeKeys({1, 2})) { + request->Record.AddKeyColumns(key); + } + return request.Release(); + }); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnSplit) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - auto delayed = ConditionalEraseRowsDelayedResolve(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - SimulateSleep(server, TDuration::Seconds(1)); - SetSplitMergePartCountLimit(&runtime, -1); - auto tabletIds = GetTableShards(server, sender, "/Root/table-1"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); - const ui64 txId = AsyncSplitTable(server, sender, "Root/table-1", tabletIds[0], 2); - WaitTxNotification(server, sender, txId); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ABORTED); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnSchemeTx) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + auto delayed = ConditionalEraseRowsDelayedResolve(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + SimulateSleep(server, TDuration::Seconds(1)); + SetSplitMergePartCountLimit(&runtime, -1); + auto tabletIds = GetTableShards(server, sender, "/Root/table-1"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + const ui64 txId = AsyncSplitTable(server, sender, "Root/table-1", tabletIds[0], 2); + WaitTxNotification(server, sender, txId); + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ABORTED); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnSchemeTx) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - auto delayed = ConditionalEraseRowsDelayedResolve(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - const ui64 txId = AsyncAlterAddExtraColumn(server, "/Root", "table-1"); - WaitTxNotification(server, sender, txId); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ERASE_ERROR); - } - - Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnDeadShard) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + auto delayed = ConditionalEraseRowsDelayedResolve(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + const ui64 txId = AsyncAlterAddExtraColumn(server, "/Root", "table-1"); + WaitTxNotification(server, sender, txId); + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ERASE_ERROR); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnDeadShard) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - ExecSQL(server, sender, R"( - UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )"); - - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - auto indexes = GetIndexes(server, sender, "/Root/table-1"); - auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", - tableId, 4, TInstant::Now().GetValue(), indexes); - - auto tabletIds = GetTableShards(server, sender, "/Root/table-1/by_skey/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + ExecSQL(server, sender, R"( + UPSERT INTO [/Root/table-1] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )"); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + auto indexes = GetIndexes(server, sender, "/Root/table-1"); + auto delayed = ConditionalEraseRowsDelayedPlan(server, sender, "/Root/table-1", + tableId, 4, TInstant::Now().GetValue(), indexes); + + auto tabletIds = GetTableShards(server, sender, "/Root/table-1/by_skey/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(tabletIds.size(), 1); RebootTablet(runtime, tabletIds[0], sender); - - for (auto& ev : delayed) { - runtime.Send(ev.Release(), 0, true); - } - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ERASE_ERROR); - } - - Y_UNIT_TEST(DistributedEraseTxShouldFailOnVariousErrors) { - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + + for (auto& ev : delayed) { + runtime.Send(ev.Release(), 0, true); + } + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::ERASE_ERROR); + } + + Y_UNIT_TEST(DistributedEraseTxShouldFailOnVariousErrors) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - CreateIndexedTable(server, sender, "/Root", "table-1"); - auto tableId = ResolveTableId(server, sender, "/Root/table-1"); - - ui64 txId = 100; - TProto::TDistributedEraseTx tx; - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(TTableId(), {1}, SerializeKeys({1, 2})); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Unknown table id"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(TTableId(tableId.PathId, 100), {1}, SerializeKeys({1, 2})); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Schema version mismatch"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {0, 1}, SerializeKeys({1, 2})); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Key column count mismatch"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {0}, SerializeKeys({1, 2})); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Key column schema mismatch"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, {"trash"}); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Cannot parse key"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, SerializeKeys({1, 2}, &KeyCellsRepeater)); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Cell count doesn't match row scheme"); - - *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, SerializeKeys({1, 2})); - // underlying request is valid now - - tx.MutableDependents()->Add(); - auto& dependency = *tx.MutableDependencies()->Add(); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "can only have dependents or dependencies"); - - tx.MutableDependents()->Clear(); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + CreateIndexedTable(server, sender, "/Root", "table-1"); + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + + ui64 txId = 100; + TProto::TDistributedEraseTx tx; + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(TTableId(), {1}, SerializeKeys({1, 2})); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Unknown table id"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(TTableId(tableId.PathId, 100), {1}, SerializeKeys({1, 2})); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Schema version mismatch"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {0, 1}, SerializeKeys({1, 2})); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Key column count mismatch"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {0}, SerializeKeys({1, 2})); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Key column schema mismatch"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, {"trash"}); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Cannot parse key"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, SerializeKeys({1, 2}, &KeyCellsRepeater)); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Cell count doesn't match row scheme"); + + *tx.MutableEraseRowsRequest() = MakeEraseRowsRequest(tableId, {1}, SerializeKeys({1, 2})); + // underlying request is valid now + + tx.MutableDependents()->Add(); + auto& dependency = *tx.MutableDependencies()->Add(); + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "can only have dependents or dependencies"); + + tx.MutableDependents()->Clear(); dependency.SetPresentRows(NDataShard::SerializeBitMap(TDynBitMap().Push(1))); - DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, - TProto::TEvProposeTxResult::BAD_REQUEST, "Present rows count mismatch"); - } - - Y_UNIT_TEST(ConditionalEraseRowsCheckLimits) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") + DistributedEraseTx(server, sender, "/Root/table-1", txId++, tx, + TProto::TEvProposeTxResult::BAD_REQUEST, "Present rows count mismatch"); + } + + Y_UNIT_TEST(ConditionalEraseRowsCheckLimits) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") .SetUseRealThreads(false); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - auto check = [&](const char* table, const TProto::TLimits& limits, ui32 expectedRuns) { - CreateIndexedTable(server, sender, "/Root", table); - ExecSQL(server, sender, Sprintf(R"( - UPSERT INTO [/Root/%s] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )", table)); - - auto tableId = ResolveTableId(server, sender, Sprintf("/Root/%s", table)); - auto indexes = GetIndexes(server, sender, Sprintf("/Root/%s", table)); - ConditionalEraseRows(server, sender, Sprintf("/Root/%s", table), tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes, limits); - - ui32 runs = 0; - TEvResponse::ProtoRecordType::EStatus status; - - do { - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - ++runs; - status = ev->Get()->Record.GetStatus(); - } while (status != TEvResponse::ProtoRecordType::OK); - - UNIT_ASSERT_VALUES_EQUAL(runs, expectedRuns); - }; - - auto makeLimits = [](ui32 maxBytes, ui32 minKeys, ui32 maxKeys) -> TProto::TLimits { - TProto::TLimits result; - - result.SetBatchMaxBytes(maxBytes); - result.SetBatchMinKeys(minKeys); - result.SetBatchMaxKeys(maxKeys); - - return result; - }; - - check("table-1", makeLimits(100, 1, 1), 4); // 3 PARTIAL (1 per key) + 1 final OK when exhausted - check("table-2", makeLimits(1, 2, 3), 2); // 1 PARTIAL (first 2 keys) + 1 final OK (last key + exhausted) - check("table-3", makeLimits(100, 1, 3), 2); // 1 PARTIAL (for 3 keys) + 1 final OK when exhausted - check("table-4", makeLimits(100, 1, 4), 1); // just final OK when exhausted - } - - Y_UNIT_TEST(ConditionalEraseRowsAsyncIndex) { - using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings - .SetDomainName("Root") - .SetUseRealThreads(false) - .SetEnableAsyncIndexes(true); - - TServer::TPtr server = new TServer(serverSettings); - auto& runtime = *server->GetRuntime(); - const TActorId sender = runtime.AllocateEdgeActor(); - - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); - InitRoot(server, sender); - - auto check = [&](const char* table, const TShardedTableOptions& opts) { - CreateShardedTable(server, sender, "/Root", table, opts); - ExecSQL(server, sender, Sprintf(R"( - UPSERT INTO [/Root/%s] (key, skey, tkey, value) VALUES - (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), - (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), - (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); - )", table)); - - auto tableId = ResolveTableId(server, sender, Sprintf("/Root/%s", table)); - auto indexes = GetIndexes(server, sender, Sprintf("/Root/%s", table)); - ConditionalEraseRows(server, sender, Sprintf("/Root/%s", table), tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes); - - auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); - - for (const auto index : opts.Indexes_) { - do { - auto content = ReadShardedTable(server, Sprintf("/Root/%s/%s/indexImplTable", table, index.Name.c_str())); - - if (index.Type == NKikimrSchemeOp::EIndexTypeGlobal) { - UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); - break; - } else if (index.Type == NKikimrSchemeOp::EIndexTypeGlobalAsync) { - if (StripInPlace(content) == "") { - break; - } - } else { - UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(index.Type)); - } - - SimulateSleep(server, TDuration::Seconds(1)); - } while (true); - } - }; - - check("table-1", TShardedTableOptions() - .EnableOutOfOrder(false) - .Columns({ - {"key", "Uint32", true, false}, - {"skey", "Uint32", false, false}, - {"tkey", "Uint32", false, false}, - {"value", "Timestamp", false, false} - }) - .Indexes({ - {"by_skey", {"skey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, - {"by_tkey", {"tkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync} - }) - ); - - check("table-2", TShardedTableOptions() - .EnableOutOfOrder(false) - .Columns({ - {"key", "Uint32", true, false}, - {"skey", "Uint32", false, false}, - {"tkey", "Uint32", false, false}, - {"value", "Timestamp", false, false} - }) - .Indexes({ - {"by_skey", {"skey"}, {}, NKikimrSchemeOp::EIndexTypeGlobal}, - {"by_tkey", {"tkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync} - }) - ); - } -} - -} // namespace NKikimr + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + auto check = [&](const char* table, const TProto::TLimits& limits, ui32 expectedRuns) { + CreateIndexedTable(server, sender, "/Root", table); + ExecSQL(server, sender, Sprintf(R"( + UPSERT INTO [/Root/%s] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )", table)); + + auto tableId = ResolveTableId(server, sender, Sprintf("/Root/%s", table)); + auto indexes = GetIndexes(server, sender, Sprintf("/Root/%s", table)); + ConditionalEraseRows(server, sender, Sprintf("/Root/%s", table), tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes, limits); + + ui32 runs = 0; + TEvResponse::ProtoRecordType::EStatus status; + + do { + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + ++runs; + status = ev->Get()->Record.GetStatus(); + } while (status != TEvResponse::ProtoRecordType::OK); + + UNIT_ASSERT_VALUES_EQUAL(runs, expectedRuns); + }; + + auto makeLimits = [](ui32 maxBytes, ui32 minKeys, ui32 maxKeys) -> TProto::TLimits { + TProto::TLimits result; + + result.SetBatchMaxBytes(maxBytes); + result.SetBatchMinKeys(minKeys); + result.SetBatchMaxKeys(maxKeys); + + return result; + }; + + check("table-1", makeLimits(100, 1, 1), 4); // 3 PARTIAL (1 per key) + 1 final OK when exhausted + check("table-2", makeLimits(1, 2, 3), 2); // 1 PARTIAL (first 2 keys) + 1 final OK (last key + exhausted) + check("table-3", makeLimits(100, 1, 3), 2); // 1 PARTIAL (for 3 keys) + 1 final OK when exhausted + check("table-4", makeLimits(100, 1, 4), 1); // just final OK when exhausted + } + + Y_UNIT_TEST(ConditionalEraseRowsAsyncIndex) { + using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") + .SetUseRealThreads(false) + .SetEnableAsyncIndexes(true); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + InitRoot(server, sender); + + auto check = [&](const char* table, const TShardedTableOptions& opts) { + CreateShardedTable(server, sender, "/Root", table, opts); + ExecSQL(server, sender, Sprintf(R"( + UPSERT INTO [/Root/%s] (key, skey, tkey, value) VALUES + (1, 10, 300, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp)), + (2, 20, 200, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp)), + (3, 30, 100, CAST("2020-04-15T00:00:00.000000Z" AS Timestamp)); + )", table)); + + auto tableId = ResolveTableId(server, sender, Sprintf("/Root/%s", table)); + auto indexes = GetIndexes(server, sender, Sprintf("/Root/%s", table)); + ConditionalEraseRows(server, sender, Sprintf("/Root/%s", table), tableId, 4, TInstant::Now().GetValue(), TUnit::AUTO, indexes); + + auto ev = server->GetRuntime()->GrabEdgeEventRethrow<TEvResponse>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), TEvResponse::ProtoRecordType::OK); + + for (const auto index : opts.Indexes_) { + do { + auto content = ReadShardedTable(server, Sprintf("/Root/%s/%s/indexImplTable", table, index.Name.c_str())); + + if (index.Type == NKikimrSchemeOp::EIndexTypeGlobal) { + UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), ""); + break; + } else if (index.Type == NKikimrSchemeOp::EIndexTypeGlobalAsync) { + if (StripInPlace(content) == "") { + break; + } + } else { + UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(index.Type)); + } + + SimulateSleep(server, TDuration::Seconds(1)); + } while (true); + } + }; + + check("table-1", TShardedTableOptions() + .EnableOutOfOrder(false) + .Columns({ + {"key", "Uint32", true, false}, + {"skey", "Uint32", false, false}, + {"tkey", "Uint32", false, false}, + {"value", "Timestamp", false, false} + }) + .Indexes({ + {"by_skey", {"skey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, + {"by_tkey", {"tkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync} + }) + ); + + check("table-2", TShardedTableOptions() + .EnableOutOfOrder(false) + .Columns({ + {"key", "Uint32", true, false}, + {"skey", "Uint32", false, false}, + {"tkey", "Uint32", false, false}, + {"value", "Timestamp", false, false} + }) + .Indexes({ + {"by_skey", {"skey"}, {}, NKikimrSchemeOp::EIndexTypeGlobal}, + {"by_tkey", {"tkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync} + }) + ); + } +} + +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_init.cpp b/ydb/core/tx/datashard/datashard_ut_init.cpp index 6c2ce8c2706..818e30ab0cb 100644 --- a/ydb/core/tx/datashard/datashard_ut_init.cpp +++ b/ydb/core/tx/datashard/datashard_ut_init.cpp @@ -72,7 +72,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardTestInit) { UNIT_ASSERT_EQUAL(datashard->GetSnapshotManager().GetMvccState(), NDataShard::EMvccState::MvccEnabled); - runtime.GetAppData().FeatureFlags.SetEnableMvccForTest(false); + runtime.GetAppData().FeatureFlags.SetEnableMvccForTest(false); GracefulRestartTablet(runtime, tabletId, sender); @@ -157,9 +157,9 @@ Y_UNIT_TEST_SUITE(TTxDataShardTestInit) { } } else if (event->GetTypeRewrite() == TEvSchemeShard::EvDescribeSchemeResult) { auto &rec = event->Get<TEvSchemeShard::TEvDescribeSchemeResult>()->GetRecord(); - const bool hasPartitioning = rec.GetPathDescription().TablePartitionsSize(); - // there are few in-flight TEvDescribeSchemeResult msgs, we need one with no partitioning - if (!hasPartitioning && rec.GetPathDescription().GetSelf().GetPathId() == tableId) { + const bool hasPartitioning = rec.GetPathDescription().TablePartitionsSize(); + // there are few in-flight TEvDescribeSchemeResult msgs, we need one with no partitioning + if (!hasPartitioning && rec.GetPathDescription().GetSelf().GetPathId() == tableId) { sawResolve = true; if (dropResolve) return TTestActorRuntime::EEventAction::DROP; diff --git a/ydb/core/tx/datashard/direct_tx_unit.cpp b/ydb/core/tx/datashard/direct_tx_unit.cpp index e2a56547a4f..1284b1f6ad6 100644 --- a/ydb/core/tx/datashard/direct_tx_unit.cpp +++ b/ydb/core/tx/datashard/direct_tx_unit.cpp @@ -1,71 +1,71 @@ #include "datashard_direct_transaction.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" #include "setup_sys_locks.h" namespace NKikimr { namespace NDataShard { -class TDirectOpUnit : public TExecutionUnit { +class TDirectOpUnit : public TExecutionUnit { public: - TDirectOpUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::DirectOp, true, self, pipeline) - { - } + TDirectOpUnit(TDataShard& self, TPipeline& pipeline) + : TExecutionUnit(EExecutionUnitKind::DirectOp, true, self, pipeline) + { + } - ~TDirectOpUnit() - { - } + ~TDirectOpUnit() + { + } - bool IsReadyToExecute(TOperation::TPtr op) const override { + bool IsReadyToExecute(TOperation::TPtr op) const override { return !op->HasRuntimeConflicts(); } - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { Y_UNUSED(ctx); - if (op->IsImmediate()) { + if (op->IsImmediate()) { // Every time we execute immediate transaction we may choose a new mvcc version - op->MvccReadWriteVersion.reset(); + op->MvccReadWriteVersion.reset(); } - TSetupSysLocks guardLocks(op, DataShard); + TSetupSysLocks guardLocks(op, DataShard); - TDirectTransaction* tx = dynamic_cast<TDirectTransaction*>(op.Get()); - Y_VERIFY(tx != nullptr); + TDirectTransaction* tx = dynamic_cast<TDirectTransaction*>(op.Get()); + Y_VERIFY(tx != nullptr); - if (!tx->Execute(&DataShard, txc)) { + if (!tx->Execute(&DataShard, txc)) { return EExecutionStatus::Restart; } - if (auto changes = tx->GetCollectedChanges()) { - op->ChangeRecords().reserve(changes.size()); - for (const auto& change : changes) { - op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); - } - } - + if (auto changes = tx->GetCollectedChanges()) { + op->ChangeRecords().reserve(changes.size()); + for (const auto& change : changes) { + op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); + } + } + DataShard.SysLocksTable().ApplyLocks(); - Pipeline.AddCommittingOp(op); + Pipeline.AddCommittingOp(op); return EExecutionStatus::DelayCompleteNoMoreRestarts; } - void Complete(TOperation::TPtr op, const TActorContext& ctx) override { - Pipeline.RemoveCommittingOp(op); - DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); + void Complete(TOperation::TPtr op, const TActorContext& ctx) override { + Pipeline.RemoveCommittingOp(op); + DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); + + TDirectTransaction* tx = dynamic_cast<TDirectTransaction*>(op.Get()); + Y_VERIFY(tx != nullptr); - TDirectTransaction* tx = dynamic_cast<TDirectTransaction*>(op.Get()); - Y_VERIFY(tx != nullptr); - - tx->SendResult(&DataShard, ctx); + tx->SendResult(&DataShard, ctx); } -}; // TDirectOpUnit - -THolder<TExecutionUnit> CreateDirectOpUnit(TDataShard& self, TPipeline& pipeline) { +}; // TDirectOpUnit + +THolder<TExecutionUnit> CreateDirectOpUnit(TDataShard& self, TPipeline& pipeline) { return THolder(new TDirectOpUnit(self, pipeline)); } } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/drop_cdc_stream_unit.cpp b/ydb/core/tx/datashard/drop_cdc_stream_unit.cpp index 8d4f15e5b33..039a379a52b 100644 --- a/ydb/core/tx/datashard/drop_cdc_stream_unit.cpp +++ b/ydb/core/tx/datashard/drop_cdc_stream_unit.cpp @@ -1,63 +1,63 @@ -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -namespace NKikimr { +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +namespace NKikimr { namespace NDataShard { - -class TDropCdcStreamUnit : public TExecutionUnit { - THolder<TEvChangeExchange::TEvRemoveSender> RemoveSender; - -public: + +class TDropCdcStreamUnit : public TExecutionUnit { + THolder<TEvChangeExchange::TEvRemoveSender> RemoveSender; + +public: TDropCdcStreamUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::DropCdcStream, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { - Y_VERIFY(op->IsSchemeTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - auto& schemeTx = tx->GetSchemeTx(); - if (!schemeTx.HasDropCdcStreamNotice()) { - return EExecutionStatus::Executed; - } - - const auto& params = schemeTx.GetDropCdcStreamNotice(); - - const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); - Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); - - const auto streamPathId = TPathId(params.GetStreamPathId().GetOwnerId(), params.GetStreamPathId().GetLocalId()); - Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); - - auto tableInfo = DataShard.AlterTableDropCdcStream(ctx, txc, pathId, params.GetTableSchemaVersion(), streamPathId); - DataShard.AddUserTable(pathId, tableInfo); - - RemoveSender.Reset(new TEvChangeExchange::TEvRemoveSender(streamPathId)); - - BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); - op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); - - return EExecutionStatus::DelayCompleteNoMoreRestarts; - } - - void Complete(TOperation::TPtr, const TActorContext& ctx) override { - if (RemoveSender) { - ctx.Send(DataShard.GetChangeSender(), RemoveSender.Release()); - } - } -}; - + : TExecutionUnit(EExecutionUnitKind::DropCdcStream, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { + Y_VERIFY(op->IsSchemeTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + auto& schemeTx = tx->GetSchemeTx(); + if (!schemeTx.HasDropCdcStreamNotice()) { + return EExecutionStatus::Executed; + } + + const auto& params = schemeTx.GetDropCdcStreamNotice(); + + const auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); + Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); + + const auto streamPathId = TPathId(params.GetStreamPathId().GetOwnerId(), params.GetStreamPathId().GetLocalId()); + Y_VERIFY(streamPathId.OwnerId == DataShard.GetPathOwnerId()); + + auto tableInfo = DataShard.AlterTableDropCdcStream(ctx, txc, pathId, params.GetTableSchemaVersion(), streamPathId); + DataShard.AddUserTable(pathId, tableInfo); + + RemoveSender.Reset(new TEvChangeExchange::TEvRemoveSender(streamPathId)); + + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); + op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); + + return EExecutionStatus::DelayCompleteNoMoreRestarts; + } + + void Complete(TOperation::TPtr, const TActorContext& ctx) override { + if (RemoveSender) { + ctx.Send(DataShard.GetChangeSender(), RemoveSender.Release()); + } + } +}; + THolder<TExecutionUnit> CreateDropCdcStreamUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TDropCdcStreamUnit(self, pipeline)); -} - + return THolder(new TDropCdcStreamUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/drop_index_notice_unit.cpp b/ydb/core/tx/datashard/drop_index_notice_unit.cpp index bd40c5f7933..43110699580 100644 --- a/ydb/core/tx/datashard/drop_index_notice_unit.cpp +++ b/ydb/core/tx/datashard/drop_index_notice_unit.cpp @@ -6,11 +6,11 @@ namespace NKikimr { namespace NDataShard { class TDropIndexNoticeUnit : public TExecutionUnit { - THolder<TEvChangeExchange::TEvRemoveSender> RemoveSender; - + THolder<TEvChangeExchange::TEvRemoveSender> RemoveSender; + public: TDropIndexNoticeUnit(TDataShard& dataShard, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::DropIndexNotice, false, dataShard, pipeline) + : TExecutionUnit(EExecutionUnitKind::DropIndexNotice, false, dataShard, pipeline) { } bool IsReadyToExecute(TOperation::TPtr) const override { @@ -33,37 +33,37 @@ public: auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); - TUserTable::TPtr tableInfo; - if (params.HasIndexPathId()) { - auto indexPathId = TPathId(params.GetIndexPathId().GetOwnerId(), params.GetIndexPathId().GetLocalId()); - - const auto& userTables = DataShard.GetUserTables(); - Y_VERIFY(userTables.contains(pathId.LocalPathId)); - const auto& indexes = userTables.at(pathId.LocalPathId)->Indexes; - - auto it = indexes.find(indexPathId); + TUserTable::TPtr tableInfo; + if (params.HasIndexPathId()) { + auto indexPathId = TPathId(params.GetIndexPathId().GetOwnerId(), params.GetIndexPathId().GetLocalId()); + + const auto& userTables = DataShard.GetUserTables(); + Y_VERIFY(userTables.contains(pathId.LocalPathId)); + const auto& indexes = userTables.at(pathId.LocalPathId)->Indexes; + + auto it = indexes.find(indexPathId); if (it != indexes.end() && it->second.Type == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync) { - RemoveSender.Reset(new TEvChangeExchange::TEvRemoveSender(indexPathId)); - } - - tableInfo = DataShard.AlterTableDropIndex(ctx, txc, pathId, params.GetTableSchemaVersion(), indexPathId); - } else { - tableInfo = DataShard.AlterTableSchemaVersion(ctx, txc, pathId, params.GetTableSchemaVersion()); - } - - Y_VERIFY(tableInfo); + RemoveSender.Reset(new TEvChangeExchange::TEvRemoveSender(indexPathId)); + } + + tableInfo = DataShard.AlterTableDropIndex(ctx, txc, pathId, params.GetTableSchemaVersion(), indexPathId); + } else { + tableInfo = DataShard.AlterTableSchemaVersion(ctx, txc, pathId, params.GetTableSchemaVersion()); + } + + Y_VERIFY(tableInfo); DataShard.AddUserTable(pathId, tableInfo); BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); - return EExecutionStatus::DelayCompleteNoMoreRestarts; + return EExecutionStatus::DelayCompleteNoMoreRestarts; } - void Complete(TOperation::TPtr, const TActorContext& ctx) override { - if (RemoveSender) { - ctx.Send(DataShard.GetChangeSender(), RemoveSender.Release()); - } + void Complete(TOperation::TPtr, const TActorContext& ctx) override { + if (RemoveSender) { + ctx.Send(DataShard.GetChangeSender(), RemoveSender.Release()); + } } }; diff --git a/ydb/core/tx/datashard/erase_rows_condition.cpp b/ydb/core/tx/datashard/erase_rows_condition.cpp index cf66fca8688..d6960773513 100644 --- a/ydb/core/tx/datashard/erase_rows_condition.cpp +++ b/ydb/core/tx/datashard/erase_rows_condition.cpp @@ -1,198 +1,198 @@ -#include "erase_rows_condition.h" - +#include "erase_rows_condition.h" + #include <ydb/library/dynumber/dynumber.h> - -#include <library/cpp/actors/core/log.h> - -#include <util/datetime/base.h> -#include <util/string/cast.h> - -namespace NKikimr { + +#include <library/cpp/actors/core/log.h> + +#include <util/datetime/base.h> +#include <util/string/cast.h> + +namespace NKikimr { namespace NDataShard { - -class TExpirationCondition: public IEraseRowsCondition { + +class TExpirationCondition: public IEraseRowsCondition { using EUnit = NKikimrSchemeOp::TTTLSettings::EUnit; - - static TMaybe<ui64> InstantValue(TInstant instant, EUnit unit) { - switch (unit) { + + static TMaybe<ui64> InstantValue(TInstant instant, EUnit unit) { + switch (unit) { case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: - return instant.Seconds(); + return instant.Seconds(); case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: - return instant.MilliSeconds(); + return instant.MilliSeconds(); case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: - return instant.MicroSeconds(); + return instant.MicroSeconds(); case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: - return instant.NanoSeconds(); - default: - return Nothing(); - } - } - - TMaybe<TString> GetWallClockDyNumber() const { - if (WallClockDyNumber) { - return WallClockDyNumber; - } - - if (CannotParseDyNumber) { - return Nothing(); - } - - const auto instantValue = InstantValue(WallClockInstant, Unit); - if (!instantValue) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unsupported unit: " << static_cast<ui32>(Unit)); - return Nothing(); - } - - const auto strInstant = ToString(*instantValue); - const auto WallClockDyNumber = NDyNumber::ParseDyNumberString(strInstant); - if (!WallClockDyNumber) { - CannotParseDyNumber = true; - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Cannot parse DyNumber from: " << strInstant.Quote()); - } - - return WallClockDyNumber; - } - - bool Check(ui64 value) const { - switch (Type) { - // 'date-type column' mode - case NScheme::NTypeIds::Date: - return TInstant::Days(value) <= WallClockInstant; - case NScheme::NTypeIds::Datetime: - return TInstant::Seconds(value) <= WallClockInstant; - case NScheme::NTypeIds::Timestamp: - return TInstant::MicroSeconds(value) <= WallClockInstant; - // 'value since epoch' mode - case NScheme::NTypeIds::Uint32: - case NScheme::NTypeIds::Uint64: - switch (Unit) { + return instant.NanoSeconds(); + default: + return Nothing(); + } + } + + TMaybe<TString> GetWallClockDyNumber() const { + if (WallClockDyNumber) { + return WallClockDyNumber; + } + + if (CannotParseDyNumber) { + return Nothing(); + } + + const auto instantValue = InstantValue(WallClockInstant, Unit); + if (!instantValue) { + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, + "Unsupported unit: " << static_cast<ui32>(Unit)); + return Nothing(); + } + + const auto strInstant = ToString(*instantValue); + const auto WallClockDyNumber = NDyNumber::ParseDyNumberString(strInstant); + if (!WallClockDyNumber) { + CannotParseDyNumber = true; + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, + "Cannot parse DyNumber from: " << strInstant.Quote()); + } + + return WallClockDyNumber; + } + + bool Check(ui64 value) const { + switch (Type) { + // 'date-type column' mode + case NScheme::NTypeIds::Date: + return TInstant::Days(value) <= WallClockInstant; + case NScheme::NTypeIds::Datetime: + return TInstant::Seconds(value) <= WallClockInstant; + case NScheme::NTypeIds::Timestamp: + return TInstant::MicroSeconds(value) <= WallClockInstant; + // 'value since epoch' mode + case NScheme::NTypeIds::Uint32: + case NScheme::NTypeIds::Uint64: + switch (Unit) { case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: - return TInstant::Seconds(value) <= WallClockInstant; + return TInstant::Seconds(value) <= WallClockInstant; case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: - return TInstant::MilliSeconds(value) <= WallClockInstant; + return TInstant::MilliSeconds(value) <= WallClockInstant; case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: - return TInstant::MicroSeconds(value) <= WallClockInstant; + return TInstant::MicroSeconds(value) <= WallClockInstant; case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: - return TInstant::MicroSeconds(value / 1000) <= WallClockInstant; - default: - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unsupported unit: " << static_cast<ui32>(Unit)); - return false; - } - default: - Y_FAIL("Unreachable"); - } - } - - bool Check(TStringBuf value) const { - switch (Type) { - // 'value since epoch' mode - case NScheme::NTypeIds::DyNumber: - if (const auto& wallClockDyNumber = GetWallClockDyNumber()) { - Y_VERIFY(NDyNumber::IsValidDyNumber(value)); - return value <= *wallClockDyNumber; - } else { - return false; - } - default: - Y_FAIL("Unreachable"); - } - } - -public: - explicit TExpirationCondition(NTable::TTag columnId, ui64 wallClockTimestamp, EUnit unit) - : ColumnId(columnId) - , WallClockInstant(TInstant::FromValue(wallClockTimestamp)) - , Unit(unit) - , CannotParseDyNumber(false) - , Pos(Max<NTable::TPos>()) - , Type(0) - { - } - - explicit TExpirationCondition(const NKikimrTxDataShard::TExpirationCondition& proto) - : TExpirationCondition(proto.GetColumnId(), proto.GetWallClockTimestamp(), proto.GetColumnUnit()) - { - } - - void AddToRequest(NKikimrTxDataShard::TEvEraseRowsRequest& request) const override { - auto& proto = *request.MutableExpiration(); - proto.SetColumnId(ColumnId); - proto.SetWallClockTimestamp(WallClockInstant.GetValue()); - proto.SetColumnUnit(Unit); - } - + return TInstant::MicroSeconds(value / 1000) <= WallClockInstant; + default: + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, + "Unsupported unit: " << static_cast<ui32>(Unit)); + return false; + } + default: + Y_FAIL("Unreachable"); + } + } + + bool Check(TStringBuf value) const { + switch (Type) { + // 'value since epoch' mode + case NScheme::NTypeIds::DyNumber: + if (const auto& wallClockDyNumber = GetWallClockDyNumber()) { + Y_VERIFY(NDyNumber::IsValidDyNumber(value)); + return value <= *wallClockDyNumber; + } else { + return false; + } + default: + Y_FAIL("Unreachable"); + } + } + +public: + explicit TExpirationCondition(NTable::TTag columnId, ui64 wallClockTimestamp, EUnit unit) + : ColumnId(columnId) + , WallClockInstant(TInstant::FromValue(wallClockTimestamp)) + , Unit(unit) + , CannotParseDyNumber(false) + , Pos(Max<NTable::TPos>()) + , Type(0) + { + } + + explicit TExpirationCondition(const NKikimrTxDataShard::TExpirationCondition& proto) + : TExpirationCondition(proto.GetColumnId(), proto.GetWallClockTimestamp(), proto.GetColumnUnit()) + { + } + + void AddToRequest(NKikimrTxDataShard::TEvEraseRowsRequest& request) const override { + auto& proto = *request.MutableExpiration(); + proto.SetColumnId(ColumnId); + proto.SetWallClockTimestamp(WallClockInstant.GetValue()); + proto.SetColumnUnit(Unit); + } + void Prepare(TIntrusiveConstPtr<NTable::TRowScheme> scheme, TMaybe<NTable::TPos> remapPos) override { - const auto* columnInfo = scheme->ColInfo(ColumnId); - Y_VERIFY(columnInfo); - - Pos = remapPos.GetOrElse(columnInfo->Pos); - Y_VERIFY(Pos < scheme->Tags().size()); - - Type = columnInfo->TypeId; - } - - bool Check(const NTable::TRowState& row) const override { - Y_VERIFY(Pos != Max<NTable::TPos>()); - Y_VERIFY(Pos < row.Size()); - - const auto& cell = row.Get(Pos); - if (cell.IsNull()) { - return false; - } - - switch (Type) { - case NScheme::NTypeIds::Date: - return Check(cell.AsValue<ui16>()); - case NScheme::NTypeIds::Datetime: - case NScheme::NTypeIds::Uint32: - return Check(cell.AsValue<ui32>()); - case NScheme::NTypeIds::Timestamp: - case NScheme::NTypeIds::Uint64: - return Check(cell.AsValue<ui64>()); - case NScheme::NTypeIds::DyNumber: - return Check(cell.AsBuf()); - default: - return false; - } - } - - TVector<NTable::TTag> Tags() const override { - return {ColumnId}; - } - -private: - const NTable::TTag ColumnId; - const TInstant WallClockInstant; - const EUnit Unit; - - mutable TMaybe<TString> WallClockDyNumber; - mutable bool CannotParseDyNumber; - - NTable::TPos Pos; - NScheme::TTypeId Type; - -}; // TExpirationCondition - -template <typename TProto> -IEraseRowsCondition* CreateEraseRowsConditionImpl(const TProto& proto) { - switch (proto.GetConditionCase()) { - case TProto::kExpiration: - return new TExpirationCondition(proto.GetExpiration()); - default: - return nullptr; - } -} - -IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvEraseRowsRequest& request) { - return CreateEraseRowsConditionImpl(request); -} - -IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& request) { - return CreateEraseRowsConditionImpl(request); -} - + const auto* columnInfo = scheme->ColInfo(ColumnId); + Y_VERIFY(columnInfo); + + Pos = remapPos.GetOrElse(columnInfo->Pos); + Y_VERIFY(Pos < scheme->Tags().size()); + + Type = columnInfo->TypeId; + } + + bool Check(const NTable::TRowState& row) const override { + Y_VERIFY(Pos != Max<NTable::TPos>()); + Y_VERIFY(Pos < row.Size()); + + const auto& cell = row.Get(Pos); + if (cell.IsNull()) { + return false; + } + + switch (Type) { + case NScheme::NTypeIds::Date: + return Check(cell.AsValue<ui16>()); + case NScheme::NTypeIds::Datetime: + case NScheme::NTypeIds::Uint32: + return Check(cell.AsValue<ui32>()); + case NScheme::NTypeIds::Timestamp: + case NScheme::NTypeIds::Uint64: + return Check(cell.AsValue<ui64>()); + case NScheme::NTypeIds::DyNumber: + return Check(cell.AsBuf()); + default: + return false; + } + } + + TVector<NTable::TTag> Tags() const override { + return {ColumnId}; + } + +private: + const NTable::TTag ColumnId; + const TInstant WallClockInstant; + const EUnit Unit; + + mutable TMaybe<TString> WallClockDyNumber; + mutable bool CannotParseDyNumber; + + NTable::TPos Pos; + NScheme::TTypeId Type; + +}; // TExpirationCondition + +template <typename TProto> +IEraseRowsCondition* CreateEraseRowsConditionImpl(const TProto& proto) { + switch (proto.GetConditionCase()) { + case TProto::kExpiration: + return new TExpirationCondition(proto.GetExpiration()); + default: + return nullptr; + } +} + +IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvEraseRowsRequest& request) { + return CreateEraseRowsConditionImpl(request); +} + +IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& request) { + return CreateEraseRowsConditionImpl(request); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/erase_rows_condition.h b/ydb/core/tx/datashard/erase_rows_condition.h index ba5c9103a50..df902660bad 100644 --- a/ydb/core/tx/datashard/erase_rows_condition.h +++ b/ydb/core/tx/datashard/erase_rows_condition.h @@ -1,29 +1,29 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/protos/tx_datashard.pb.h> #include <ydb/core/tablet_flat/flat_row_scheme.h> #include <ydb/core/tablet_flat/flat_row_state.h> - -#include <util/generic/maybe.h> -#include <util/generic/vector.h> - -namespace NKikimr { + +#include <util/generic/maybe.h> +#include <util/generic/vector.h> + +namespace NKikimr { namespace NDataShard { - -class IEraseRowsCondition { -public: - virtual ~IEraseRowsCondition() = default; - virtual void AddToRequest(NKikimrTxDataShard::TEvEraseRowsRequest& request) const = 0; + +class IEraseRowsCondition { +public: + virtual ~IEraseRowsCondition() = default; + virtual void AddToRequest(NKikimrTxDataShard::TEvEraseRowsRequest& request) const = 0; virtual void Prepare(TIntrusiveConstPtr<NTable::TRowScheme> scheme, TMaybe<NTable::TPos> remapPos = Nothing()) = 0; - virtual bool Check(const NTable::TRowState& row) const = 0; - virtual TVector<NTable::TTag> Tags() const = 0; - -}; // IEraseRowsCondition - -IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvEraseRowsRequest& request); -IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& request); - + virtual bool Check(const NTable::TRowState& row) const = 0; + virtual TVector<NTable::TTag> Tags() const = 0; + +}; // IEraseRowsCondition + +IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvEraseRowsRequest& request); +IEraseRowsCondition* CreateEraseRowsCondition(const NKikimrTxDataShard::TEvConditionalEraseRowsRequest& request); + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/execute_data_tx_unit.cpp b/ydb/core/tx/datashard/execute_data_tx_unit.cpp index 749e1954991..b32f635cc0f 100644 --- a/ydb/core/tx/datashard/execute_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/execute_data_tx_unit.cpp @@ -239,13 +239,13 @@ void TExecuteDataTxUnit::ExecuteDataTx(TOperation::TPtr op, if (op->IsImmediate() && !op->IsReadOnly()) DataShard.PromoteCompleteEdge(writeVersion.Step, txc); - - if (auto changes = tx->GetDataTx()->GetCollectedChanges()) { - op->ChangeRecords().reserve(changes.size()); - for (const auto& change : changes) { - op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); - } - } + + if (auto changes = tx->GetDataTx()->GetCollectedChanges()) { + op->ChangeRecords().reserve(changes.size()); + for (const auto& change : changes) { + op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); + } + } } LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, diff --git a/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp b/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp index 79c16e27d1c..3c9a86e06b4 100644 --- a/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp +++ b/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp @@ -1,149 +1,149 @@ -#include "change_collector.h" -#include "datashard_active_transaction.h" -#include "datashard_distributed_erase.h" -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" -#include "setup_sys_locks.h" - -#include <util/generic/bitmap.h> - -namespace NKikimr { +#include "change_collector.h" +#include "datashard_active_transaction.h" +#include "datashard_distributed_erase.h" +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" +#include "setup_sys_locks.h" + +#include <util/generic/bitmap.h> + +namespace NKikimr { namespace NDataShard { - -class TExecuteDistributedEraseTxUnit : public TExecutionUnit { - using IChangeCollector = NMiniKQL::IChangeCollector; - -public: + +class TExecuteDistributedEraseTxUnit : public TExecutionUnit { + using IChangeCollector = NMiniKQL::IChangeCollector; + +public: TExecuteDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::ExecuteDistributedEraseTx, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr op) const override { + : TExecutionUnit(EExecutionUnitKind::ExecuteDistributedEraseTx, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr op) const override { if (DataShard.IsStopping()) { // Avoid doing any new work when datashard is stopping return false; } - return !op->HasRuntimeConflicts(); - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext&) override { - Y_VERIFY(op->IsDistributedEraseTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - TSetupSysLocks guardLocks(op, DataShard); - - const auto& eraseTx = tx->GetDistributedEraseTx(); - const auto& request = eraseTx->GetRequest(); - const auto versions = DataShard.GetReadWriteVersions(op.Get()); - - if (eraseTx->HasDependents()) { - THolder<IChangeCollector> changeCollector{CreateChangeCollector(DataShard, txc.DB, request.GetTableId(), false)}; - - changeCollector->SetWriteVersion(versions.WriteVersion); - if (changeCollector->NeedToReadKeys()) { - changeCollector->SetReadVersion(versions.ReadVersion); - } - - auto presentRows = TDynBitMap().Set(0, request.KeyColumnsSize()); - if (!Execute(txc, request, presentRows, eraseTx->GetConfirmedRows(), versions.WriteVersion, changeCollector.Get())) { - return EExecutionStatus::Restart; - } - - if (auto changes = changeCollector->GetCollected()) { - op->ChangeRecords().reserve(changes.size()); - for (const auto& change : changes) { - op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); - } - } - } else if (eraseTx->HasDependencies()) { - THashMap<ui64, TDynBitMap> presentRows; - for (const auto& dependency : eraseTx->GetDependencies()) { - Y_VERIFY(!presentRows.contains(dependency.GetShardId())); - presentRows.emplace(dependency.GetShardId(), DeserializeBitMap<TDynBitMap>(dependency.GetPresentRows())); - } - - for (const auto& [_, readSets] : op->InReadSets()) { - for (const auto& rs : readSets) { - NKikimrTxDataShard::TDistributedEraseRS body; - Y_VERIFY(body.ParseFromArray(rs.Body.data(), rs.Body.size())); - - Y_VERIFY(presentRows.contains(rs.Origin)); - const bool ok = Execute(txc, request, presentRows.at(rs.Origin), DeserializeBitMap<TDynBitMap>(body.GetConfirmedRows()), versions.WriteVersion); - Y_VERIFY(ok); - } - } - } else { - Y_FAIL_S("Invalid distributed erase tx: " << eraseTx->GetBody().ShortDebugString()); - } - - BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); - DataShard.SysLocksTable().ApplyLocks(); + return !op->HasRuntimeConflicts(); + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext&) override { + Y_VERIFY(op->IsDistributedEraseTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + TSetupSysLocks guardLocks(op, DataShard); + + const auto& eraseTx = tx->GetDistributedEraseTx(); + const auto& request = eraseTx->GetRequest(); + const auto versions = DataShard.GetReadWriteVersions(op.Get()); + + if (eraseTx->HasDependents()) { + THolder<IChangeCollector> changeCollector{CreateChangeCollector(DataShard, txc.DB, request.GetTableId(), false)}; + + changeCollector->SetWriteVersion(versions.WriteVersion); + if (changeCollector->NeedToReadKeys()) { + changeCollector->SetReadVersion(versions.ReadVersion); + } + + auto presentRows = TDynBitMap().Set(0, request.KeyColumnsSize()); + if (!Execute(txc, request, presentRows, eraseTx->GetConfirmedRows(), versions.WriteVersion, changeCollector.Get())) { + return EExecutionStatus::Restart; + } + + if (auto changes = changeCollector->GetCollected()) { + op->ChangeRecords().reserve(changes.size()); + for (const auto& change : changes) { + op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); + } + } + } else if (eraseTx->HasDependencies()) { + THashMap<ui64, TDynBitMap> presentRows; + for (const auto& dependency : eraseTx->GetDependencies()) { + Y_VERIFY(!presentRows.contains(dependency.GetShardId())); + presentRows.emplace(dependency.GetShardId(), DeserializeBitMap<TDynBitMap>(dependency.GetPresentRows())); + } + + for (const auto& [_, readSets] : op->InReadSets()) { + for (const auto& rs : readSets) { + NKikimrTxDataShard::TDistributedEraseRS body; + Y_VERIFY(body.ParseFromArray(rs.Body.data(), rs.Body.size())); + + Y_VERIFY(presentRows.contains(rs.Origin)); + const bool ok = Execute(txc, request, presentRows.at(rs.Origin), DeserializeBitMap<TDynBitMap>(body.GetConfirmedRows()), versions.WriteVersion); + Y_VERIFY(ok); + } + } + } else { + Y_FAIL_S("Invalid distributed erase tx: " << eraseTx->GetBody().ShortDebugString()); + } + + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE); + DataShard.SysLocksTable().ApplyLocks(); Pipeline.AddCommittingOp(op); - - return EExecutionStatus::ExecutedNoMoreRestarts; - } - - bool Execute(TTransactionContext& txc, const NKikimrTxDataShard::TEvEraseRowsRequest& request, - const TDynBitMap& presentRows, const TDynBitMap& confirmedRows, const TRowVersion& writeVersion, - IChangeCollector* changeCollector = nullptr) - { - const ui64 tableId = request.GetTableId(); - const TTableId fullTableId(DataShard.GetPathOwnerId(), tableId); - - Y_VERIFY(DataShard.GetUserTables().contains(tableId)); - const TUserTable& tableInfo = *DataShard.GetUserTables().at(tableId); - - size_t row = 0; - bool pageFault = false; - Y_FOR_EACH_BIT(i, presentRows) { - if (!confirmedRows.Test(i)) { - ++row; - continue; - } - - const auto& serializedKey = request.GetKeyColumns(row++); - TSerializedCellVec keyCells; - Y_VERIFY(TSerializedCellVec::TryParse(serializedKey, keyCells)); - Y_VERIFY(keyCells.GetCells().size() == tableInfo.KeyColumnTypes.size()); - - TVector<TRawTypeValue> key; - for (size_t ki : xrange(tableInfo.KeyColumnTypes.size())) { - const auto& kt = tableInfo.KeyColumnTypes[ki]; - const TCell& cell = keyCells.GetCells()[ki]; - - key.emplace_back(TRawTypeValue(cell.AsRef(), kt)); - } - - if (changeCollector) { + + return EExecutionStatus::ExecutedNoMoreRestarts; + } + + bool Execute(TTransactionContext& txc, const NKikimrTxDataShard::TEvEraseRowsRequest& request, + const TDynBitMap& presentRows, const TDynBitMap& confirmedRows, const TRowVersion& writeVersion, + IChangeCollector* changeCollector = nullptr) + { + const ui64 tableId = request.GetTableId(); + const TTableId fullTableId(DataShard.GetPathOwnerId(), tableId); + + Y_VERIFY(DataShard.GetUserTables().contains(tableId)); + const TUserTable& tableInfo = *DataShard.GetUserTables().at(tableId); + + size_t row = 0; + bool pageFault = false; + Y_FOR_EACH_BIT(i, presentRows) { + if (!confirmedRows.Test(i)) { + ++row; + continue; + } + + const auto& serializedKey = request.GetKeyColumns(row++); + TSerializedCellVec keyCells; + Y_VERIFY(TSerializedCellVec::TryParse(serializedKey, keyCells)); + Y_VERIFY(keyCells.GetCells().size() == tableInfo.KeyColumnTypes.size()); + + TVector<TRawTypeValue> key; + for (size_t ki : xrange(tableInfo.KeyColumnTypes.size())) { + const auto& kt = tableInfo.KeyColumnTypes[ki]; + const TCell& cell = keyCells.GetCells()[ki]; + + key.emplace_back(TRawTypeValue(cell.AsRef(), kt)); + } + + if (changeCollector) { if (!changeCollector->Collect(fullTableId, NTable::ERowOp::Erase, key, {})) { - changeCollector->Reset(); - pageFault = true; - } - } - - if (pageFault) { - continue; - } - - DataShard.SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); + changeCollector->Reset(); + pageFault = true; + } + } + + if (pageFault) { + continue; + } + + DataShard.SysLocksTable().BreakLock(fullTableId, keyCells.GetCells()); txc.DB.Update(tableInfo.LocalTid, NTable::ERowOp::Erase, key, {}, writeVersion); - } - - return !pageFault; - } - - void Complete(TOperation::TPtr, const TActorContext&) override { - } -}; - + } + + return !pageFault; + } + + void Complete(TOperation::TPtr, const TActorContext&) override { + } +}; + THolder<TExecutionUnit> CreateExecuteDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TExecuteDistributedEraseTxUnit(self, pipeline)); -} - + return THolder(new TExecuteDistributedEraseTxUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp b/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp index b730e72f732..70e7be3662b 100644 --- a/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp @@ -155,13 +155,13 @@ EExecutionStatus TExecuteKqpDataTxUnit::Execute(TOperation::TPtr op, TTransactio DataShard.PromoteCompleteEdge(writeVersion.Step, txc); } - if (auto changes = dataTx->GetCollectedChanges()) { - op->ChangeRecords().reserve(changes.size()); - for (const auto& change : changes) { - op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); - } - } - + if (auto changes = dataTx->GetCollectedChanges()) { + op->ChangeRecords().reserve(changes.size()); + for (const auto& change : changes) { + op->ChangeRecords().emplace_back(change.Order(), change.PathId(), change.BodySize()); + } + } + KqpUpdateDataShardStatCounters(DataShard, dataTx->GetCounters()); auto statsMode = kqpTx.GetRuntimeSettings().GetStatsMode(); if (statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) { diff --git a/ydb/core/tx/datashard/execution_unit.cpp b/ydb/core/tx/datashard/execution_unit.cpp index ade9bf51a1f..9f6e13fa5c7 100644 --- a/ydb/core/tx/datashard/execution_unit.cpp +++ b/ydb/core/tx/datashard/execution_unit.cpp @@ -16,8 +16,8 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateCheckSchemeTxUnit(dataShard, pipeline); case EExecutionUnitKind::CheckSnapshotTx: return CreateCheckSnapshotTxUnit(dataShard, pipeline); - case EExecutionUnitKind::CheckDistributedEraseTx: - return CreateCheckDistributedEraseTxUnit(dataShard, pipeline); + case EExecutionUnitKind::CheckDistributedEraseTx: + return CreateCheckDistributedEraseTxUnit(dataShard, pipeline); case EExecutionUnitKind::CheckCommitWritesTx: return CreateCheckCommitWritesTxUnit(dataShard, pipeline); case EExecutionUnitKind::StoreDataTx: @@ -26,8 +26,8 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateStoreSchemeTxUnit(dataShard, pipeline); case EExecutionUnitKind::StoreSnapshotTx: return CreateStoreSnapshotTxUnit(dataShard, pipeline); - case EExecutionUnitKind::StoreDistributedEraseTx: - return CreateStoreDistributedEraseTxUnit(dataShard, pipeline); + case EExecutionUnitKind::StoreDistributedEraseTx: + return CreateStoreDistributedEraseTxUnit(dataShard, pipeline); case EExecutionUnitKind::StoreCommitWritesTx: return CreateStoreCommitWritesTxUnit(dataShard, pipeline); case EExecutionUnitKind::BuildAndWaitDependencies: @@ -48,8 +48,8 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateProtectSchemeEchoesUnit(dataShard, pipeline); case EExecutionUnitKind::BuildDataTxOutRS: return CreateBuildDataTxOutRSUnit(dataShard, pipeline); - case EExecutionUnitKind::BuildDistributedEraseTxOutRS: - return CreateBuildDistributedEraseTxOutRSUnit(dataShard, pipeline); + case EExecutionUnitKind::BuildDistributedEraseTxOutRS: + return CreateBuildDistributedEraseTxOutRSUnit(dataShard, pipeline); case EExecutionUnitKind::BuildKqpDataTxOutRS: return CreateBuildKqpDataTxOutRSUnit(dataShard, pipeline); case EExecutionUnitKind::StoreAndSendOutRS: @@ -58,16 +58,16 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreatePrepareDataTxInRSUnit(dataShard, pipeline); case EExecutionUnitKind::PrepareKqpDataTxInRS: return CreatePrepareKqpDataTxInRSUnit(dataShard, pipeline); - case EExecutionUnitKind::PrepareDistributedEraseTxInRS: - return CreatePrepareDistributedEraseTxInRSUnit(dataShard, pipeline); + case EExecutionUnitKind::PrepareDistributedEraseTxInRS: + return CreatePrepareDistributedEraseTxInRSUnit(dataShard, pipeline); case EExecutionUnitKind::LoadAndWaitInRS: return CreateLoadAndWaitInRSUnit(dataShard, pipeline); case EExecutionUnitKind::ExecuteDataTx: return CreateExecuteDataTxUnit(dataShard, pipeline); case EExecutionUnitKind::ExecuteKqpDataTx: return CreateExecuteKqpDataTxUnit(dataShard, pipeline); - case EExecutionUnitKind::ExecuteDistributedEraseTx: - return CreateExecuteDistributedEraseTxUnit(dataShard, pipeline); + case EExecutionUnitKind::ExecuteDistributedEraseTx: + return CreateExecuteDistributedEraseTxUnit(dataShard, pipeline); case EExecutionUnitKind::ExecuteCommitWritesTx: return CreateExecuteCommitWritesTxUnit(dataShard, pipeline); case EExecutionUnitKind::CompleteOperation: @@ -86,10 +86,10 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateBuildSchemeTxOutRSUnit(dataShard, pipeline); case EExecutionUnitKind::PrepareSchemeTxInRS: return CreatePrepareSchemeTxInRSUnit(dataShard, pipeline); - case EExecutionUnitKind::Backup: - return CreateBackupUnit(dataShard, pipeline); - case EExecutionUnitKind::Restore: - return CreateRestoreUnit(dataShard, pipeline); + case EExecutionUnitKind::Backup: + return CreateBackupUnit(dataShard, pipeline); + case EExecutionUnitKind::Restore: + return CreateRestoreUnit(dataShard, pipeline); case EExecutionUnitKind::CreateTable: return CreateCreateTableUnit(dataShard, pipeline); case EExecutionUnitKind::ReceiveSnapshot: @@ -100,8 +100,8 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateAlterTableUnit(dataShard, pipeline); case EExecutionUnitKind::DropTable: return CreateDropTableUnit(dataShard, pipeline); - case EExecutionUnitKind::DirectOp: - return CreateDirectOpUnit(dataShard, pipeline); + case EExecutionUnitKind::DirectOp: + return CreateDirectOpUnit(dataShard, pipeline); case EExecutionUnitKind::CreatePersistentSnapshot: return CreateCreatePersistentSnapshotUnit(dataShard, pipeline); case EExecutionUnitKind::DropPersistentSnapshot: @@ -118,12 +118,12 @@ THolder<TExecutionUnit> CreateExecutionUnit(EExecutionUnitKind kind, return CreateDropIndexNoticeUnit(dataShard, pipeline); case EExecutionUnitKind::MoveTable: return CreateMoveTableUnit(dataShard, pipeline); - case EExecutionUnitKind::CreateCdcStream: - return CreateCreateCdcStreamUnit(dataShard, pipeline); - case EExecutionUnitKind::AlterCdcStream: - return CreateAlterCdcStreamUnit(dataShard, pipeline); - case EExecutionUnitKind::DropCdcStream: - return CreateDropCdcStreamUnit(dataShard, pipeline); + case EExecutionUnitKind::CreateCdcStream: + return CreateCreateCdcStreamUnit(dataShard, pipeline); + case EExecutionUnitKind::AlterCdcStream: + return CreateAlterCdcStreamUnit(dataShard, pipeline); + case EExecutionUnitKind::DropCdcStream: + return CreateDropCdcStreamUnit(dataShard, pipeline); default: Y_FAIL_S("Unexpected execution kind " << kind << " (" << (ui32)kind << ")"); } @@ -195,19 +195,19 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& return true; } - if (!op->IsReadOnly() && DataShard.CheckChangesQueueOverflow()) { - TString err = TStringBuilder() - << "Can't execute at blocked shard: " << " tablet id: " << DataShard.TabletID(); - BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED) - ->AddError(NKikimrTxDataShard::TError::SHARD_IS_BLOCKED, err); - - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, - "Tablet " << DataShard.TabletID() << " rejecting tx due to changes queue overflow"); - - op->Abort(); - return true; - } - + if (!op->IsReadOnly() && DataShard.CheckChangesQueueOverflow()) { + TString err = TStringBuilder() + << "Can't execute at blocked shard: " << " tablet id: " << DataShard.TabletID(); + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED) + ->AddError(NKikimrTxDataShard::TError::SHARD_IS_BLOCKED, err); + + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, + "Tablet " << DataShard.TabletID() << " rejecting tx due to changes queue overflow"); + + op->Abort(); + return true; + } + return false; } @@ -236,10 +236,10 @@ bool TExecutionUnit::WillRejectDataTx(TOperation::TPtr op) const { return true; } - if (!op->IsReadOnly() && DataShard.CheckChangesQueueOverflow()) { - return true; - } - + if (!op->IsReadOnly() && DataShard.CheckChangesQueueOverflow()) { + return true; + } + return false; } diff --git a/ydb/core/tx/datashard/execution_unit_ctors.h b/ydb/core/tx/datashard/execution_unit_ctors.h index 93effadc422..f043a02fad1 100644 --- a/ydb/core/tx/datashard/execution_unit_ctors.h +++ b/ydb/core/tx/datashard/execution_unit_ctors.h @@ -62,6 +62,6 @@ THolder<TExecutionUnit> CreateMoveTableUnit(TDataShard &dataShard, TPipeline &pi THolder<TExecutionUnit> CreateCreateCdcStreamUnit(TDataShard &dataShard, TPipeline &pipeline); THolder<TExecutionUnit> CreateAlterCdcStreamUnit(TDataShard &dataShard, TPipeline &pipeline); THolder<TExecutionUnit> CreateDropCdcStreamUnit(TDataShard &dataShard, TPipeline &pipeline); - + } // namespace NDataShard } // namespace NKikimr diff --git a/ydb/core/tx/datashard/execution_unit_kind.h b/ydb/core/tx/datashard/execution_unit_kind.h index eb12d0d827a..88e763887f6 100644 --- a/ydb/core/tx/datashard/execution_unit_kind.h +++ b/ydb/core/tx/datashard/execution_unit_kind.h @@ -8,12 +8,12 @@ enum class EExecutionUnitKind : ui32 { CheckDataTx, CheckSchemeTx, CheckSnapshotTx, - CheckDistributedEraseTx, + CheckDistributedEraseTx, CheckCommitWritesTx, StoreDataTx, StoreSchemeTx, StoreSnapshotTx, - StoreDistributedEraseTx, + StoreDistributedEraseTx, StoreCommitWritesTx, BuildAndWaitDependencies, FinishPropose, @@ -25,15 +25,15 @@ enum class EExecutionUnitKind : ui32 { ProtectSchemeEchoes, BuildDataTxOutRS, BuildKqpDataTxOutRS, - BuildDistributedEraseTxOutRS, + BuildDistributedEraseTxOutRS, StoreAndSendOutRS, PrepareDataTxInRS, PrepareKqpDataTxInRS, - PrepareDistributedEraseTxInRS, + PrepareDistributedEraseTxInRS, LoadAndWaitInRS, ExecuteDataTx, ExecuteKqpDataTx, - ExecuteDistributedEraseTx, + ExecuteDistributedEraseTx, ExecuteCommitWritesTx, CompleteOperation, ExecuteKqpScanTx, @@ -43,14 +43,14 @@ enum class EExecutionUnitKind : ui32 { MakeSnapshot, BuildSchemeTxOutRS, PrepareSchemeTxInRS, - Backup, - Restore, + Backup, + Restore, CreateTable, ReceiveSnapshot, AlterMoveShadow, AlterTable, DropTable, - DirectOp, + DirectOp, CreatePersistentSnapshot, DropPersistentSnapshot, CreateVolatileSnapshot, @@ -59,9 +59,9 @@ enum class EExecutionUnitKind : ui32 { FinalizeBuildIndex, DropIndexNotice, MoveTable, - CreateCdcStream, - AlterCdcStream, - DropCdcStream, + CreateCdcStream, + AlterCdcStream, + DropCdcStream, Count, Unspecified }; diff --git a/ydb/core/tx/datashard/export_common.cpp b/ydb/core/tx/datashard/export_common.cpp index 62046a34cef..e00cc2b5601 100644 --- a/ydb/core/tx/datashard/export_common.cpp +++ b/ydb/core/tx/datashard/export_common.cpp @@ -1,86 +1,86 @@ -#include "export_common.h" - +#include "export_common.h" + #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/ydb_convert/table_description.h> #include <ydb/core/ydb_convert/ydb_convert.h> #include <ydb/library/dynumber/dynumber.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> - + #include <ydb/library/yql/public/decimal/yql_decimal.h> - -#include <util/generic/algorithm.h> - -namespace NKikimr { + +#include <util/generic/algorithm.h> + +namespace NKikimr { namespace NDataShard { - -static void ResortColumns( - google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& columns, - const TMap<ui32, TUserTable::TUserColumn>& order) -{ - THashMap<TString, ui32> nameToTag; - for (const auto& [tag, column] : order) { - Y_VERIFY(nameToTag.emplace(column.Name, tag).second); - } - - SortBy(columns, [&nameToTag](const auto& column) { - auto it = nameToTag.find(column.name()); - Y_VERIFY(it != nameToTag.end()); - return it->second; - }); -} - -TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme( - const TMap<ui32, TUserTable::TUserColumn>& columns, + +static void ResortColumns( + google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& columns, + const TMap<ui32, TUserTable::TUserColumn>& order) +{ + THashMap<TString, ui32> nameToTag; + for (const auto& [tag, column] : order) { + Y_VERIFY(nameToTag.emplace(column.Name, tag).second); + } + + SortBy(columns, [&nameToTag](const auto& column) { + auto it = nameToTag.find(column.name()); + Y_VERIFY(it != nameToTag.end()); + return it->second; + }); +} + +TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme( + const TMap<ui32, TUserTable::TUserColumn>& columns, const NKikimrSchemeOp::TPathDescription& pathDesc) -{ - if (!pathDesc.HasTable()) { - return Nothing(); - } - - Ydb::Table::CreateTableRequest scheme; - - const auto& tableDesc = pathDesc.GetTable(); - NKikimrMiniKQL::TType mkqlKeyType; - - try { - FillColumnDescription(scheme, mkqlKeyType, tableDesc); - } catch (const yexception&) { - return Nothing(); - } - - ResortColumns(*scheme.mutable_columns(), columns); - - scheme.mutable_primary_key()->CopyFrom(tableDesc.GetKeyColumnNames()); - - try { - FillTableBoundary(scheme, tableDesc, mkqlKeyType); - } catch (const yexception&) { - return Nothing(); - } - - FillIndexDescription(scheme, tableDesc); - FillStorageSettings(scheme, tableDesc); - FillColumnFamilies(scheme, tableDesc); - FillAttributes(scheme, pathDesc); - FillPartitioningSettings(scheme, tableDesc); - FillKeyBloomFilter(scheme, tableDesc); - FillReadReplicasSettings(scheme, tableDesc); - - return scheme; -} - -TString DecimalToString(const std::pair<ui64, i64>& loHi) { - using namespace NYql::NDecimal; - - TInt128 val = FromHalfs(loHi.first, loHi.second); - return ToString(val, NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE); -} - -TString DyNumberToString(TStringBuf data) { - auto result = NDyNumber::DyNumberToString(data); - Y_VERIFY(result.Defined(), "Invalid DyNumber binary representation"); - return *result; -} - +{ + if (!pathDesc.HasTable()) { + return Nothing(); + } + + Ydb::Table::CreateTableRequest scheme; + + const auto& tableDesc = pathDesc.GetTable(); + NKikimrMiniKQL::TType mkqlKeyType; + + try { + FillColumnDescription(scheme, mkqlKeyType, tableDesc); + } catch (const yexception&) { + return Nothing(); + } + + ResortColumns(*scheme.mutable_columns(), columns); + + scheme.mutable_primary_key()->CopyFrom(tableDesc.GetKeyColumnNames()); + + try { + FillTableBoundary(scheme, tableDesc, mkqlKeyType); + } catch (const yexception&) { + return Nothing(); + } + + FillIndexDescription(scheme, tableDesc); + FillStorageSettings(scheme, tableDesc); + FillColumnFamilies(scheme, tableDesc); + FillAttributes(scheme, pathDesc); + FillPartitioningSettings(scheme, tableDesc); + FillKeyBloomFilter(scheme, tableDesc); + FillReadReplicasSettings(scheme, tableDesc); + + return scheme; +} + +TString DecimalToString(const std::pair<ui64, i64>& loHi) { + using namespace NYql::NDecimal; + + TInt128 val = FromHalfs(loHi.first, loHi.second); + return ToString(val, NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE); +} + +TString DyNumberToString(TStringBuf data) { + auto result = NDyNumber::DyNumberToString(data); + Y_VERIFY(result.Defined(), "Invalid DyNumber binary representation"); + return *result; +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/export_common.h b/ydb/core/tx/datashard/export_common.h index 9aab2bdbbe7..b9ccffe2a1a 100644 --- a/ydb/core/tx/datashard/export_common.h +++ b/ydb/core/tx/datashard/export_common.h @@ -1,42 +1,42 @@ -#pragma once - -#include "datashard_user_table.h" - +#pragma once + +#include "datashard_user_table.h" + #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> - -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -#if defined EXPORT_LOG_T || \ - defined EXPORT_LOG_D || \ - defined EXPORT_LOG_I || \ - defined EXPORT_LOG_N || \ - defined EXPORT_LOG_W || \ - defined EXPORT_LOG_E || \ - defined EXPORT_LOG_C -#error log macro redefinition -#endif - -#define EXPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) - -namespace NKikimr { + +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +#if defined EXPORT_LOG_T || \ + defined EXPORT_LOG_D || \ + defined EXPORT_LOG_I || \ + defined EXPORT_LOG_N || \ + defined EXPORT_LOG_W || \ + defined EXPORT_LOG_E || \ + defined EXPORT_LOG_C +#error log macro redefinition +#endif + +#define EXPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) +#define EXPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) + +namespace NKikimr { namespace NDataShard { - -TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme( - const TMap<ui32, TUserTable::TUserColumn>& columns, + +TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme( + const TMap<ui32, TUserTable::TUserColumn>& columns, const NKikimrSchemeOp::TPathDescription& pathDesc); - -TString DecimalToString(const std::pair<ui64, i64>& loHi); -TString DyNumberToString(TStringBuf data); - + +TString DecimalToString(const std::pair<ui64, i64>& loHi); +TString DyNumberToString(TStringBuf data); + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/export_iface.h b/ydb/core/tx/datashard/export_iface.h index ae617aa4bf1..2374a9ab8b6 100644 --- a/ydb/core/tx/datashard/export_iface.h +++ b/ydb/core/tx/datashard/export_iface.h @@ -8,30 +8,30 @@ namespace NDataShard { class IExport { public: - using TTableColumns = TMap<ui32, TUserTable::TUserColumn>; - using TTask = NKikimrSchemeOp::TBackupTask; - using IBuffer = NExportScan::IBuffer; - -public: - virtual ~IExport() = default; - - virtual IActor* CreateUploader( - const TActorId& dataShard, - ui64 txId, - const TTableColumns& columns, - const TTask& task) const = 0; - - virtual IBuffer* CreateBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) const = 0; - + using TTableColumns = TMap<ui32, TUserTable::TUserColumn>; + using TTask = NKikimrSchemeOp::TBackupTask; + using IBuffer = NExportScan::IBuffer; + +public: + virtual ~IExport() = default; + + virtual IActor* CreateUploader( + const TActorId& dataShard, + ui64 txId, + const TTableColumns& columns, + const TTask& task) const = 0; + + virtual IBuffer* CreateBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) const = 0; + virtual void Shutdown() const = 0; }; class IExportFactory { public: - virtual ~IExportFactory() = default; - - virtual IExport* CreateExportToYt(bool useTypeV3) const = 0; - virtual IExport* CreateExportToS3() const = 0; + virtual ~IExportFactory() = default; + + virtual IExport* CreateExportToYt(bool useTypeV3) const = 0; + virtual IExport* CreateExportToS3() const = 0; virtual void Shutdown() = 0; }; diff --git a/ydb/core/tx/datashard/export_s3.h b/ydb/core/tx/datashard/export_s3.h index 5ed0487c2d6..95cb9de7643 100644 --- a/ydb/core/tx/datashard/export_s3.h +++ b/ydb/core/tx/datashard/export_s3.h @@ -1,30 +1,30 @@ -#pragma once - -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "defs.h" +#pragma once + +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "defs.h" #include "export_iface.h" -#include "export_s3_buffer.h" - -namespace NKikimr { +#include "export_s3_buffer.h" + +namespace NKikimr { namespace NDataShard { - -class TS3Export: public IExport { + +class TS3Export: public IExport { public: IActor* CreateUploader( - const TActorId& dataShard, - ui64 txId, - const TTableColumns& columns, - const TTask& task) const override; - - IBuffer* CreateBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) const override { - return CreateS3ExportBuffer(columns, rowsLimit, bytesLimit); - } + const TActorId& dataShard, + ui64 txId, + const TTableColumns& columns, + const TTask& task) const override; + + IBuffer* CreateBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) const override { + return CreateS3ExportBuffer(columns, rowsLimit, bytesLimit); + } void Shutdown() const override {} }; - + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/export_s3_base_uploader.h b/ydb/core/tx/datashard/export_s3_base_uploader.h index 3bb4323641c..8c72a8c3936 100644 --- a/ydb/core/tx/datashard/export_s3_base_uploader.h +++ b/ydb/core/tx/datashard/export_s3_base_uploader.h @@ -1,467 +1,467 @@ #pragma once -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "datashard.h" -#include "export_common.h" -#include "export_s3.h" -#include "s3_common.h" - +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "datashard.h" +#include "export_common.h" +#include "export_s3.h" +#include "s3_common.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/wrappers/s3_wrapper.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -using namespace Aws::S3; -using namespace Aws; - -class IProxyOps { -public: - virtual ~IProxyOps() = default; - virtual bool NeedToResolveProxy() const = 0; - virtual void ResolveProxy() = 0; - -}; // IProxyOps - -template <typename TDerived> -class TS3UploaderBase: public TActorBootstrapped<TDerived> - , private NWrappers::TS3User - , public IProxyOps -{ - using TEvS3Wrapper = NWrappers::TEvS3Wrapper; - using TEvBuffer = TEvExportScan::TEvBuffer<TString>; - -protected: - void Restart() { - Y_VERIFY(ProxyResolved); - - MultiPart = false; - Last = false; - Parts.clear(); - - if (Attempt) { - this->Send(std::exchange(Client, TActorId()), new TEvents::TEvPoisonPill()); - } - - Client = this->RegisterWithSameMailbox(NWrappers::CreateS3Wrapper(Settings.Credentials, Settings.Config)); - - if (!SchemeUploaded) { - this->Become(&TDerived::StateUploadScheme); - - UploadScheme(); - } else { - this->Become(&TDerived::StateUploadData); - - if (Attempt) { - this->Send(std::exchange(Scanner, TActorId()), new TEvExportScan::TEvReset()); - } else if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - } - } - - void UploadScheme() { - Y_VERIFY(!SchemeUploaded); - - if (!Scheme) { - return Finish(false, "Cannot infer scheme"); - } - - google::protobuf::TextFormat::PrintToString(Scheme.GetRef(), &Buffer); - - auto request = Model::PutObjectRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.SchemeKey) - .WithStorageClass(Settings.StorageClass); - this->Send(Client, new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(Buffer))); - } - - void HandleScheme(TEvS3Wrapper::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("HandleScheme TEvS3Wrapper::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - + +using namespace Aws::S3; +using namespace Aws; + +class IProxyOps { +public: + virtual ~IProxyOps() = default; + virtual bool NeedToResolveProxy() const = 0; + virtual void ResolveProxy() = 0; + +}; // IProxyOps + +template <typename TDerived> +class TS3UploaderBase: public TActorBootstrapped<TDerived> + , private NWrappers::TS3User + , public IProxyOps +{ + using TEvS3Wrapper = NWrappers::TEvS3Wrapper; + using TEvBuffer = TEvExportScan::TEvBuffer<TString>; + +protected: + void Restart() { + Y_VERIFY(ProxyResolved); + + MultiPart = false; + Last = false; + Parts.clear(); + + if (Attempt) { + this->Send(std::exchange(Client, TActorId()), new TEvents::TEvPoisonPill()); + } + + Client = this->RegisterWithSameMailbox(NWrappers::CreateS3Wrapper(Settings.Credentials, Settings.Config)); + + if (!SchemeUploaded) { + this->Become(&TDerived::StateUploadScheme); + + UploadScheme(); + } else { + this->Become(&TDerived::StateUploadData); + + if (Attempt) { + this->Send(std::exchange(Scanner, TActorId()), new TEvExportScan::TEvReset()); + } else if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + } + + void UploadScheme() { + Y_VERIFY(!SchemeUploaded); + + if (!Scheme) { + return Finish(false, "Cannot infer scheme"); + } + + google::protobuf::TextFormat::PrintToString(Scheme.GetRef(), &Buffer); + + auto request = Model::PutObjectRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.SchemeKey) + .WithStorageClass(Settings.StorageClass); + this->Send(Client, new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(Buffer))); + } + + void HandleScheme(TEvS3Wrapper::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("HandleScheme TEvS3Wrapper::TEvPutObjectResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + if (!CheckResult(result, TStringBuf("PutObject (scheme)"))) { - return; - } - - SchemeUploaded = true; - - if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - - this->Become(&TDerived::StateUploadData); - } - - void Handle(TEvExportScan::TEvReady::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvReady" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender); - - Scanner = ev->Sender; - - if (Error) { - return PassAway(); - } - - if (ProxyResolved && SchemeUploaded) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - } - - void Handle(TEvBuffer::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", msg# " << ev->Get()->ToString()); - - if (ev->Sender != Scanner) { - EXPORT_LOG_W("Received buffer from unknown scanner" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", scanner# " << Scanner); - return; - } - - Last = ev->Get()->Last; - MultiPart = MultiPart || !Last; - Buffer.swap(ev->Get()->Buffer); - - UploadData(); - } - - void UploadData() { - if (!MultiPart) { - auto request = Model::PutObjectRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.DataKey) - .WithStorageClass(Settings.StorageClass); - this->Send(Client, new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(Buffer))); - } else { - if (!UploadId) { - this->Send(DataShard, new TEvDataShard::TEvGetS3Upload(this->SelfId(), TxId)); - return; - } - - auto request = Model::UploadPartRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.DataKey) - .WithUploadId(*UploadId) - .WithPartNumber(Parts.size() + 1); - this->Send(Client, new TEvS3Wrapper::TEvUploadPartRequest(request, std::move(Buffer))); - } - } - - void HandleData(TEvS3Wrapper::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("HandleData TEvS3Wrapper::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - + return; + } + + SchemeUploaded = true; + + if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + + this->Become(&TDerived::StateUploadData); + } + + void Handle(TEvExportScan::TEvReady::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvReady" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender); + + Scanner = ev->Sender; + + if (Error) { + return PassAway(); + } + + if (ProxyResolved && SchemeUploaded) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + + void Handle(TEvBuffer::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender + << ", msg# " << ev->Get()->ToString()); + + if (ev->Sender != Scanner) { + EXPORT_LOG_W("Received buffer from unknown scanner" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender + << ", scanner# " << Scanner); + return; + } + + Last = ev->Get()->Last; + MultiPart = MultiPart || !Last; + Buffer.swap(ev->Get()->Buffer); + + UploadData(); + } + + void UploadData() { + if (!MultiPart) { + auto request = Model::PutObjectRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.DataKey) + .WithStorageClass(Settings.StorageClass); + this->Send(Client, new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(Buffer))); + } else { + if (!UploadId) { + this->Send(DataShard, new TEvDataShard::TEvGetS3Upload(this->SelfId(), TxId)); + return; + } + + auto request = Model::UploadPartRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.DataKey) + .WithUploadId(*UploadId) + .WithPartNumber(Parts.size() + 1); + this->Send(Client, new TEvS3Wrapper::TEvUploadPartRequest(request, std::move(Buffer))); + } + } + + void HandleData(TEvS3Wrapper::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("HandleData TEvS3Wrapper::TEvPutObjectResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + if (!CheckResult(result, TStringBuf("PutObject (data)"))) { - return; - } - - Finish(); - } - - void Handle(TEvDataShard::TEvS3Upload::TPtr& ev) { - auto& upload = ev->Get()->Upload; - - EXPORT_LOG_D("Handle TEvDataShard::TEvS3Upload" - << ": self# " << this->SelfId() - << ", upload# " << upload); - - if (!upload) { - auto request = Model::CreateMultipartUploadRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.DataKey) - .WithStorageClass(Settings.StorageClass); - this->Send(Client, new TEvS3Wrapper::TEvCreateMultipartUploadRequest(request)); - } else { - UploadId = upload->Id; - - switch (upload->Status) { - case TS3Upload::EStatus::UploadParts: - return UploadData(); - - case TS3Upload::EStatus::Complete: { - Parts = std::move(upload->Parts); - - TVector<Model::CompletedPart> parts(Reserve(Parts.size())); - for (ui32 partIndex = 0; partIndex < Parts.size(); ++partIndex) { - parts.emplace_back(Model::CompletedPart() - .WithPartNumber(partIndex + 1) - .WithETag(Parts.at(partIndex))); - } - - auto request = Model::CompleteMultipartUploadRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.DataKey) - .WithUploadId(*UploadId) - .WithMultipartUpload(Model::CompletedMultipartUpload().WithParts(std::move(parts))); - this->Send(Client, new TEvS3Wrapper::TEvCompleteMultipartUploadRequest(request)); - break; - } - - case TS3Upload::EStatus::Abort: { - Error = std::move(upload->Error); - if (!Error) { - Error = "<empty>"; - } - - auto request = Model::AbortMultipartUploadRequest() - .WithBucket(Settings.Bucket) - .WithKey(Settings.DataKey) - .WithUploadId(*UploadId); - this->Send(Client, new TEvS3Wrapper::TEvAbortMultipartUploadRequest(request)); - break; - } - } - } - } - - void Handle(TEvS3Wrapper::TEvCreateMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvS3Wrapper::TEvCreateMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - + return; + } + + Finish(); + } + + void Handle(TEvDataShard::TEvS3Upload::TPtr& ev) { + auto& upload = ev->Get()->Upload; + + EXPORT_LOG_D("Handle TEvDataShard::TEvS3Upload" + << ": self# " << this->SelfId() + << ", upload# " << upload); + + if (!upload) { + auto request = Model::CreateMultipartUploadRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.DataKey) + .WithStorageClass(Settings.StorageClass); + this->Send(Client, new TEvS3Wrapper::TEvCreateMultipartUploadRequest(request)); + } else { + UploadId = upload->Id; + + switch (upload->Status) { + case TS3Upload::EStatus::UploadParts: + return UploadData(); + + case TS3Upload::EStatus::Complete: { + Parts = std::move(upload->Parts); + + TVector<Model::CompletedPart> parts(Reserve(Parts.size())); + for (ui32 partIndex = 0; partIndex < Parts.size(); ++partIndex) { + parts.emplace_back(Model::CompletedPart() + .WithPartNumber(partIndex + 1) + .WithETag(Parts.at(partIndex))); + } + + auto request = Model::CompleteMultipartUploadRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.DataKey) + .WithUploadId(*UploadId) + .WithMultipartUpload(Model::CompletedMultipartUpload().WithParts(std::move(parts))); + this->Send(Client, new TEvS3Wrapper::TEvCompleteMultipartUploadRequest(request)); + break; + } + + case TS3Upload::EStatus::Abort: { + Error = std::move(upload->Error); + if (!Error) { + Error = "<empty>"; + } + + auto request = Model::AbortMultipartUploadRequest() + .WithBucket(Settings.Bucket) + .WithKey(Settings.DataKey) + .WithUploadId(*UploadId); + this->Send(Client, new TEvS3Wrapper::TEvAbortMultipartUploadRequest(request)); + break; + } + } + } + } + + void Handle(TEvS3Wrapper::TEvCreateMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvS3Wrapper::TEvCreateMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + if (!CheckResult(result, TStringBuf("CreateMultipartUpload"))) { - return; - } - - this->Send(DataShard, new TEvDataShard::TEvStoreS3UploadId(this->SelfId(), TxId, result.GetResult().GetUploadId().c_str())); - } - - void Handle(TEvS3Wrapper::TEvUploadPartResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvS3Wrapper::TEvUploadPartResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - + return; + } + + this->Send(DataShard, new TEvDataShard::TEvStoreS3UploadId(this->SelfId(), TxId, result.GetResult().GetUploadId().c_str())); + } + + void Handle(TEvS3Wrapper::TEvUploadPartResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvS3Wrapper::TEvUploadPartResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + if (!CheckResult(result, TStringBuf("UploadPart"))) { - return; - } - - Parts.push_back(result.GetResult().GetETag().c_str()); - - if (Last) { - return Finish(); - } - - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - - void Handle(TEvS3Wrapper::TEvCompleteMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvS3Wrapper::TEvCompleteMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!result.IsSuccess()) { - const auto& error = result.GetError(); - if (error.GetErrorType() != S3Errors::NO_SUCH_UPLOAD) { - Error = error.GetMessage().c_str(); - } - } - - PassAway(); - } - - void Handle(TEvS3Wrapper::TEvAbortMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvS3Wrapper::TEvAbortMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!result.IsSuccess()) { - Y_VERIFY(Error); - Error = TStringBuilder() << *Error << " Additionally, 'AbortMultipartUpload' has failed: " - << result.GetError().GetMessage(); - } - - PassAway(); - } - - template <typename TResult> - bool CheckResult(const TResult& result, const TStringBuf marker) { - if (result.IsSuccess()) { - return true; - } - - EXPORT_LOG_E("Error at '" << marker << "'" - << ": self# " << this->SelfId() - << ", error# " << result); - RetryOrFinish(result.GetError()); - - return false; - } - - void RetryOrFinish(const S3Error& error) { - if (Attempt++ < Retries && error.ShouldRetry()) { - Delay = Min(Delay * Attempt, TDuration::Minutes(10)); - const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); - - this->Schedule(Delay + random, new TEvents::TEvWakeup()); - } else { - Finish(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); - } - } - - void Finish(bool success = true, const TString& error = TString()) { - EXPORT_LOG_I("Finish" - << ": self# " << this->SelfId() - << ", success# " << success - << ", error# " << error - << ", multipart# " << MultiPart - << ", uploadId# " << UploadId); - - if (!success) { - Error = error; - } - - if (!MultiPart || !UploadId) { - if (!Scanner) { - return; - } - - PassAway(); - } else { - if (success) { - this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, - TS3Upload::EStatus::Complete, std::move(Parts))); - } else { - this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, - TS3Upload::EStatus::Abort, *Error)); - } - } - } - - void PassAway() override { - if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFinish(Error.Empty(), Error.GetOrElse(TString()))); - } - - this->Send(Client, new TEvents::TEvPoisonPill()); - - IActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::EXPORT_S3_UPLOADER_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { + return; + } + + Parts.push_back(result.GetResult().GetETag().c_str()); + + if (Last) { + return Finish(); + } + + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + + void Handle(TEvS3Wrapper::TEvCompleteMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvS3Wrapper::TEvCompleteMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!result.IsSuccess()) { + const auto& error = result.GetError(); + if (error.GetErrorType() != S3Errors::NO_SUCH_UPLOAD) { + Error = error.GetMessage().c_str(); + } + } + + PassAway(); + } + + void Handle(TEvS3Wrapper::TEvAbortMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvS3Wrapper::TEvAbortMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!result.IsSuccess()) { + Y_VERIFY(Error); + Error = TStringBuilder() << *Error << " Additionally, 'AbortMultipartUpload' has failed: " + << result.GetError().GetMessage(); + } + + PassAway(); + } + + template <typename TResult> + bool CheckResult(const TResult& result, const TStringBuf marker) { + if (result.IsSuccess()) { + return true; + } + + EXPORT_LOG_E("Error at '" << marker << "'" + << ": self# " << this->SelfId() + << ", error# " << result); + RetryOrFinish(result.GetError()); + + return false; + } + + void RetryOrFinish(const S3Error& error) { + if (Attempt++ < Retries && error.ShouldRetry()) { + Delay = Min(Delay * Attempt, TDuration::Minutes(10)); + const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); + + this->Schedule(Delay + random, new TEvents::TEvWakeup()); + } else { + Finish(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); + } + } + + void Finish(bool success = true, const TString& error = TString()) { + EXPORT_LOG_I("Finish" + << ": self# " << this->SelfId() + << ", success# " << success + << ", error# " << error + << ", multipart# " << MultiPart + << ", uploadId# " << UploadId); + + if (!success) { + Error = error; + } + + if (!MultiPart || !UploadId) { + if (!Scanner) { + return; + } + + PassAway(); + } else { + if (success) { + this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, + TS3Upload::EStatus::Complete, std::move(Parts))); + } else { + this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, + TS3Upload::EStatus::Abort, *Error)); + } + } + } + + void PassAway() override { + if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFinish(Error.Empty(), Error.GetOrElse(TString()))); + } + + this->Send(Client, new TEvents::TEvPoisonPill()); + + IActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::EXPORT_S3_UPLOADER_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { return "s3"sv; - } - - explicit TS3UploaderBase( - const TActorId& dataShard, ui64 txId, + } + + explicit TS3UploaderBase( + const TActorId& dataShard, ui64 txId, const NKikimrSchemeOp::TBackupTask& task, - TMaybe<Ydb::Table::CreateTableRequest>&& scheme) - : Settings(TS3Settings::FromBackupTask(task)) - , DataShard(dataShard) - , TxId(txId) - , Scheme(std::move(scheme)) - , Retries(task.GetNumberOfRetries()) - , Attempt(0) - , Delay(TDuration::Minutes(1)) - , SchemeUploaded(task.GetShardNum() == 0 ? false : true) - { - } - - void Bootstrap() { - EXPORT_LOG_D("Bootstrap" - << ": self# " << this->SelfId() - << ", attempt# " << Attempt); - - ProxyResolved = !NeedToResolveProxy(); - if (!ProxyResolved) { - ResolveProxy(); - } else { - Restart(); - } - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExportScan::TEvReady, Handle); - - sFunc(TEvents::TEvWakeup, Bootstrap); - sFunc(TEvents::TEvPoisonPill, PassAway); - } - } - - STATEFN(StateUploadScheme) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvS3Wrapper::TEvPutObjectResponse, HandleScheme); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - STATEFN(StateUploadData) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBuffer, Handle); - hFunc(TEvDataShard::TEvS3Upload, Handle); - - hFunc(TEvS3Wrapper::TEvPutObjectResponse, HandleData); - hFunc(TEvS3Wrapper::TEvCreateMultipartUploadResponse, Handle); - hFunc(TEvS3Wrapper::TEvUploadPartResponse, Handle); - hFunc(TEvS3Wrapper::TEvCompleteMultipartUploadResponse, Handle); - hFunc(TEvS3Wrapper::TEvAbortMultipartUploadResponse, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -protected: - TS3Settings Settings; - bool ProxyResolved; - -private: - const TActorId DataShard; - const ui64 TxId; - const TMaybe<Ydb::Table::CreateTableRequest> Scheme; - - const ui32 Retries; - ui32 Attempt; - - TActorId Client; - TDuration Delay; - bool SchemeUploaded; - bool MultiPart; - bool Last; - + TMaybe<Ydb::Table::CreateTableRequest>&& scheme) + : Settings(TS3Settings::FromBackupTask(task)) + , DataShard(dataShard) + , TxId(txId) + , Scheme(std::move(scheme)) + , Retries(task.GetNumberOfRetries()) + , Attempt(0) + , Delay(TDuration::Minutes(1)) + , SchemeUploaded(task.GetShardNum() == 0 ? false : true) + { + } + + void Bootstrap() { + EXPORT_LOG_D("Bootstrap" + << ": self# " << this->SelfId() + << ", attempt# " << Attempt); + + ProxyResolved = !NeedToResolveProxy(); + if (!ProxyResolved) { + ResolveProxy(); + } else { + Restart(); + } + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExportScan::TEvReady, Handle); + + sFunc(TEvents::TEvWakeup, Bootstrap); + sFunc(TEvents::TEvPoisonPill, PassAway); + } + } + + STATEFN(StateUploadScheme) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvS3Wrapper::TEvPutObjectResponse, HandleScheme); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + STATEFN(StateUploadData) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBuffer, Handle); + hFunc(TEvDataShard::TEvS3Upload, Handle); + + hFunc(TEvS3Wrapper::TEvPutObjectResponse, HandleData); + hFunc(TEvS3Wrapper::TEvCreateMultipartUploadResponse, Handle); + hFunc(TEvS3Wrapper::TEvUploadPartResponse, Handle); + hFunc(TEvS3Wrapper::TEvCompleteMultipartUploadResponse, Handle); + hFunc(TEvS3Wrapper::TEvAbortMultipartUploadResponse, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + +protected: + TS3Settings Settings; + bool ProxyResolved; + +private: + const TActorId DataShard; + const ui64 TxId; + const TMaybe<Ydb::Table::CreateTableRequest> Scheme; + + const ui32 Retries; + ui32 Attempt; + + TActorId Client; + TDuration Delay; + bool SchemeUploaded; + bool MultiPart; + bool Last; + TActorId Scanner; - TString Buffer; - - TMaybe<TString> UploadId; - TVector<TString> Parts; - TMaybe<TString> Error; - -}; // TS3UploaderBase - + TString Buffer; + + TMaybe<TString> UploadId; + TVector<TString> Parts; + TMaybe<TString> Error; + +}; // TS3UploaderBase + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/export_s3_buffer.cpp b/ydb/core/tx/datashard/export_s3_buffer.cpp index 1d8002f6e04..470ed22d6b6 100644 --- a/ydb/core/tx/datashard/export_s3_buffer.cpp +++ b/ydb/core/tx/datashard/export_s3_buffer.cpp @@ -1,192 +1,192 @@ -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "export_common.h" -#include "export_s3.h" -#include "export_s3_buffer.h" - +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "export_common.h" +#include "export_s3.h" +#include "export_s3_buffer.h" + #include <ydb/core/tablet_flat/flat_row_state.h> #include <ydb/library/binary_json/read.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> #include <library/cpp/string_utils/quote/quote.h> - -#include <util/datetime/base.h> - -namespace NKikimr { + +#include <util/datetime/base.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NExportScan; -using TTableColumns = TS3Export::TTableColumns; - -class TS3Buffer: public IBuffer { - using TTagToColumn = TTableColumns; - using TTagToIndex = THashMap<ui32, ui32>; // index in IScan::TRow - -public: - explicit TS3Buffer(const TTagToColumn& columns, ui64 rowsLimit, ui64 bytesLimit) - : Columns(columns) - , RowsLimit(rowsLimit) - , BytesLimit(bytesLimit) - , Rows(0) - , BytesRead(0) - { - } - - void ColumnsOrder(const TVector<ui32>& tags) override { - Y_VERIFY(tags.size() == Columns.size()); - - Indices.clear(); - for (ui32 i = 0; i < tags.size(); ++i) { - const ui32 tag = tags.at(i); - auto it = Columns.find(tag); - Y_VERIFY(it != Columns.end()); - Y_VERIFY(Indices.emplace(tag, i).second); - } - } - - void Collect(const NTable::IScan::TRow& row) override { - TStringOutput stream(Buffer); - - bool needsComma = false; - for (const auto& [tag, column] : Columns) { - auto it = Indices.find(tag); - Y_VERIFY(it != Indices.end()); - Y_VERIFY(it->second < (*row).size()); - const auto& cell = (*row)[it->second]; - - BytesRead += cell.Size(); - - if (needsComma) { - stream << ","; - } else { - needsComma = true; - } - - if (cell.IsNull()) { - stream << "null"; - continue; - } - - switch (column.Type) { - case NScheme::NTypeIds::Int32: - stream << cell.AsValue<i32>(); - break; - case NScheme::NTypeIds::Uint32: - stream << cell.AsValue<ui32>(); - break; - case NScheme::NTypeIds::Int64: - stream << cell.AsValue<i64>(); - break; - case NScheme::NTypeIds::Uint64: - stream << cell.AsValue<ui64>(); - break; - case NScheme::NTypeIds::Uint8: - //case NScheme::NTypeIds::Byte: - stream << static_cast<ui32>(cell.AsValue<ui8>()); - break; - case NScheme::NTypeIds::Int8: - stream << static_cast<i32>(cell.AsValue<i8>()); - break; - case NScheme::NTypeIds::Int16: - stream << cell.AsValue<i16>(); - break; - case NScheme::NTypeIds::Uint16: - stream << cell.AsValue<ui16>(); - break; - case NScheme::NTypeIds::Bool: - stream << cell.AsValue<bool>(); - break; - case NScheme::NTypeIds::Double: - stream << cell.AsValue<double>(); - break; - case NScheme::NTypeIds::Float: - stream << cell.AsValue<float>(); - break; - case NScheme::NTypeIds::Date: - stream << TInstant::Days(cell.AsValue<ui16>()); - break; - case NScheme::NTypeIds::Datetime: - stream << TInstant::Seconds(cell.AsValue<ui32>()); - break; - case NScheme::NTypeIds::Timestamp: - stream << TInstant::MicroSeconds(cell.AsValue<ui64>()); - break; - case NScheme::NTypeIds::Interval: - stream << cell.AsValue<i64>(); - break; - case NScheme::NTypeIds::Decimal: - stream << DecimalToString(cell.AsValue<std::pair<ui64, i64>>()); - break; - case NScheme::NTypeIds::DyNumber: - stream << DyNumberToString(cell.AsBuf()); - break; - case NScheme::NTypeIds::String: - case NScheme::NTypeIds::String4k: - case NScheme::NTypeIds::String2m: - case NScheme::NTypeIds::Utf8: - case NScheme::NTypeIds::Json: - case NScheme::NTypeIds::Yson: - stream << '"' << CGIEscapeRet(cell.AsBuf()) << '"'; - break; - case NScheme::NTypeIds::JsonDocument: - stream << '"' << CGIEscapeRet(NBinaryJson::SerializeToJson(cell.AsBuf())) << '"'; - break; - default: - Y_FAIL("Unsupported type"); - }; - } - - stream << "\n"; - ++Rows; - } - - IEventBase* PrepareEvent(bool last) override { - return new TEvExportScan::TEvBuffer<TString>(Flush(), last); - } - - void Clear() override { - Flush(); - } - - TString Flush() { - Rows = 0; - BytesRead = 0; - return std::exchange(Buffer, TString()); - } - - bool IsFilled() const override { - return GetRows() >= RowsLimit || GetBytesSent() >= BytesLimit; - } - - ui64 GetRows() const override { - return Rows; - } - - ui64 GetBytesRead() const override { - return BytesRead; - } - - ui64 GetBytesSent() const override { - return Buffer.size(); - } - -private: - const TTagToColumn Columns; - const ui64 RowsLimit; - const ui64 BytesLimit; - - TTagToIndex Indices; - - ui64 Rows; - ui64 BytesRead; - TString Buffer; - -}; // TS3Buffer - -IBuffer* CreateS3ExportBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) { - return new TS3Buffer(columns, rowsLimit, bytesLimit); -} - + +using namespace NExportScan; +using TTableColumns = TS3Export::TTableColumns; + +class TS3Buffer: public IBuffer { + using TTagToColumn = TTableColumns; + using TTagToIndex = THashMap<ui32, ui32>; // index in IScan::TRow + +public: + explicit TS3Buffer(const TTagToColumn& columns, ui64 rowsLimit, ui64 bytesLimit) + : Columns(columns) + , RowsLimit(rowsLimit) + , BytesLimit(bytesLimit) + , Rows(0) + , BytesRead(0) + { + } + + void ColumnsOrder(const TVector<ui32>& tags) override { + Y_VERIFY(tags.size() == Columns.size()); + + Indices.clear(); + for (ui32 i = 0; i < tags.size(); ++i) { + const ui32 tag = tags.at(i); + auto it = Columns.find(tag); + Y_VERIFY(it != Columns.end()); + Y_VERIFY(Indices.emplace(tag, i).second); + } + } + + void Collect(const NTable::IScan::TRow& row) override { + TStringOutput stream(Buffer); + + bool needsComma = false; + for (const auto& [tag, column] : Columns) { + auto it = Indices.find(tag); + Y_VERIFY(it != Indices.end()); + Y_VERIFY(it->second < (*row).size()); + const auto& cell = (*row)[it->second]; + + BytesRead += cell.Size(); + + if (needsComma) { + stream << ","; + } else { + needsComma = true; + } + + if (cell.IsNull()) { + stream << "null"; + continue; + } + + switch (column.Type) { + case NScheme::NTypeIds::Int32: + stream << cell.AsValue<i32>(); + break; + case NScheme::NTypeIds::Uint32: + stream << cell.AsValue<ui32>(); + break; + case NScheme::NTypeIds::Int64: + stream << cell.AsValue<i64>(); + break; + case NScheme::NTypeIds::Uint64: + stream << cell.AsValue<ui64>(); + break; + case NScheme::NTypeIds::Uint8: + //case NScheme::NTypeIds::Byte: + stream << static_cast<ui32>(cell.AsValue<ui8>()); + break; + case NScheme::NTypeIds::Int8: + stream << static_cast<i32>(cell.AsValue<i8>()); + break; + case NScheme::NTypeIds::Int16: + stream << cell.AsValue<i16>(); + break; + case NScheme::NTypeIds::Uint16: + stream << cell.AsValue<ui16>(); + break; + case NScheme::NTypeIds::Bool: + stream << cell.AsValue<bool>(); + break; + case NScheme::NTypeIds::Double: + stream << cell.AsValue<double>(); + break; + case NScheme::NTypeIds::Float: + stream << cell.AsValue<float>(); + break; + case NScheme::NTypeIds::Date: + stream << TInstant::Days(cell.AsValue<ui16>()); + break; + case NScheme::NTypeIds::Datetime: + stream << TInstant::Seconds(cell.AsValue<ui32>()); + break; + case NScheme::NTypeIds::Timestamp: + stream << TInstant::MicroSeconds(cell.AsValue<ui64>()); + break; + case NScheme::NTypeIds::Interval: + stream << cell.AsValue<i64>(); + break; + case NScheme::NTypeIds::Decimal: + stream << DecimalToString(cell.AsValue<std::pair<ui64, i64>>()); + break; + case NScheme::NTypeIds::DyNumber: + stream << DyNumberToString(cell.AsBuf()); + break; + case NScheme::NTypeIds::String: + case NScheme::NTypeIds::String4k: + case NScheme::NTypeIds::String2m: + case NScheme::NTypeIds::Utf8: + case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::Yson: + stream << '"' << CGIEscapeRet(cell.AsBuf()) << '"'; + break; + case NScheme::NTypeIds::JsonDocument: + stream << '"' << CGIEscapeRet(NBinaryJson::SerializeToJson(cell.AsBuf())) << '"'; + break; + default: + Y_FAIL("Unsupported type"); + }; + } + + stream << "\n"; + ++Rows; + } + + IEventBase* PrepareEvent(bool last) override { + return new TEvExportScan::TEvBuffer<TString>(Flush(), last); + } + + void Clear() override { + Flush(); + } + + TString Flush() { + Rows = 0; + BytesRead = 0; + return std::exchange(Buffer, TString()); + } + + bool IsFilled() const override { + return GetRows() >= RowsLimit || GetBytesSent() >= BytesLimit; + } + + ui64 GetRows() const override { + return Rows; + } + + ui64 GetBytesRead() const override { + return BytesRead; + } + + ui64 GetBytesSent() const override { + return Buffer.size(); + } + +private: + const TTagToColumn Columns; + const ui64 RowsLimit; + const ui64 BytesLimit; + + TTagToIndex Indices; + + ui64 Rows; + ui64 BytesRead; + TString Buffer; + +}; // TS3Buffer + +IBuffer* CreateS3ExportBuffer(const TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit) { + return new TS3Buffer(columns, rowsLimit, bytesLimit); +} + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/export_s3_buffer.h b/ydb/core/tx/datashard/export_s3_buffer.h index f146d3e41b1..f616126dbed 100644 --- a/ydb/core/tx/datashard/export_s3_buffer.h +++ b/ydb/core/tx/datashard/export_s3_buffer.h @@ -1,16 +1,16 @@ -#pragma once - -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "export_iface.h" -#include "export_scan.h" - -namespace NKikimr { -namespace NDataShard { - -NExportScan::IBuffer* CreateS3ExportBuffer(const IExport::TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit); - -} // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +#pragma once + +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "export_iface.h" +#include "export_scan.h" + +namespace NKikimr { +namespace NDataShard { + +NExportScan::IBuffer* CreateS3ExportBuffer(const IExport::TTableColumns& columns, ui64 rowsLimit, ui64 bytesLimit); + +} // NDataShard +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/export_s3_uploader.cpp b/ydb/core/tx/datashard/export_s3_uploader.cpp index 3b9d48963da..db48213401d 100644 --- a/ydb/core/tx/datashard/export_s3_uploader.cpp +++ b/ydb/core/tx/datashard/export_s3_uploader.cpp @@ -1,39 +1,39 @@ -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "export_s3_base_uploader.h" - -namespace NKikimr { +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "export_s3_base_uploader.h" + +namespace NKikimr { namespace NDataShard { - -class TS3Uploader: public TS3UploaderBase<TS3Uploader> { -protected: - bool NeedToResolveProxy() const override { - return false; - } - - void ResolveProxy() override { - Y_FAIL("unreachable"); - } - -public: - using TS3UploaderBase::TS3UploaderBase; - -}; // TS3Uploader - + +class TS3Uploader: public TS3UploaderBase<TS3Uploader> { +protected: + bool NeedToResolveProxy() const override { + return false; + } + + void ResolveProxy() override { + Y_FAIL("unreachable"); + } + +public: + using TS3UploaderBase::TS3UploaderBase; + +}; // TS3Uploader + IActor* TS3Export::CreateUploader( const TActorId& dataShard, ui64 txId, - const TTableColumns& columns, - const TTask& task) const -{ - auto scheme = (task.GetShardNum() == 0) - ? GenYdbScheme(columns, task.GetTable()) - : Nothing(); - - return new TS3Uploader(dataShard, txId, task, std::move(scheme)); -} - + const TTableColumns& columns, + const TTask& task) const +{ + auto scheme = (task.GetShardNum() == 0) + ? GenYdbScheme(columns, task.GetTable()) + : Nothing(); + + return new TS3Uploader(dataShard, txId, task, std::move(scheme)); +} + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/export_scan.cpp b/ydb/core/tx/datashard/export_scan.cpp index 31067362294..2cda75fdd07 100644 --- a/ydb/core/tx/datashard/export_scan.cpp +++ b/ydb/core/tx/datashard/export_scan.cpp @@ -1,253 +1,253 @@ -#include "export_common.h" -#include "export_scan.h" - +#include "export_common.h" +#include "export_scan.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/tablet_flat/flat_row_state.h> #include <ydb/core/tablet_flat/flat_scan_spent.h> #include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/bitmap.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/bitmap.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NActors; -using namespace NExportScan; -using namespace NTable; - -class TExportScan: private NActors::IActor, public NTable::IScan { - enum EStateBits { - ES_REGISTERED = 0, // Actor is registered - ES_INITIALIZED, // Seek(...) was called - ES_UPLOADER_READY, - ES_BUFFER_SENT, - ES_NO_MORE_DATA, - - ES_COUNT, - }; - - struct TStats { - ui64 Rows; - ui64 BytesRead; - ui64 BytesSent; - - TStats() - : Rows(0) - , BytesRead(0) - , BytesSent(0) - { - auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "store_to_yt"); - - MonRows = counters->GetCounter("Rows", true); - MonBytesRead = counters->GetCounter("BytesRead", true); - MonBytesSent = counters->GetCounter("BytesSent", true); - } - - void Aggr(ui64 rows, ui64 bytesRead, ui64 bytesSent) { - Rows += rows; - BytesRead += bytesRead; - BytesSent += bytesSent; - - *MonRows += rows; - *MonBytesRead += bytesRead; - *MonBytesSent += bytesSent; - } - - void Aggr(IBuffer const* buffer) { - Aggr(buffer->GetRows(), buffer->GetBytesRead(), buffer->GetBytesSent()); - } - - TString ToString() const { - return TStringBuilder() - << "Stats { " - << " Rows: " << Rows - << " BytesRead: " << BytesRead - << " BytesSent: " << BytesSent - << " }"; - } - - private: - NMonitoring::TDynamicCounters::TCounterPtr MonRows; - NMonitoring::TDynamicCounters::TCounterPtr MonBytesRead; - NMonitoring::TDynamicCounters::TCounterPtr MonBytesSent; - }; - - bool IsReady() const { - return State.Test(ES_REGISTERED) && State.Test(ES_INITIALIZED); - } - - void MaybeReady() { - if (IsReady()) { - Send(Uploader, new TEvExportScan::TEvReady()); - } - } - - EScan MaybeSendBuffer() { - const bool noMoreData = State.Test(ES_NO_MORE_DATA); - - if (!noMoreData && !Buffer->IsFilled()) { - return EScan::Feed; - } - - if (!State.Test(ES_UPLOADER_READY) || State.Test(ES_BUFFER_SENT)) { - Spent->Alter(false); - return EScan::Sleep; - } - - State.Set(ES_BUFFER_SENT); - Stats->Aggr(Buffer.Get()); - Send(Uploader, Buffer->PrepareEvent(noMoreData)); - - if (noMoreData) { - Spent->Alter(false); - return EScan::Sleep; - } - - return EScan::Feed; - } - - void Handle(TEvExportScan::TEvReset::TPtr&) { - Y_VERIFY(IsReady()); - - EXPORT_LOG_D("Handle TEvExportScan::TEvReset" - << ": self# " << SelfId()); - - Stats.Reset(new TStats); - State.Reset(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT).Reset(ES_NO_MORE_DATA); - Spent->Alter(true); - Driver->Touch(EScan::Reset); - } - - void Handle(TEvExportScan::TEvFeed::TPtr&) { - Y_VERIFY(IsReady()); - - EXPORT_LOG_D("Handle TEvExportScan::TEvFeed" - << ": self# " << SelfId()); - - State.Set(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT); - Spent->Alter(true); - if (EScan::Feed == MaybeSendBuffer()) { - Driver->Touch(EScan::Feed); - } - } - - void Handle(TEvExportScan::TEvFinish::TPtr& ev) { - Y_VERIFY(IsReady()); - - EXPORT_LOG_D("Handle TEvExportScan::TEvFinish" - << ": self# " << SelfId() - << ", msg# " << ev->Get()->ToString()); - - Success = ev->Get()->Success; - Error = ev->Get()->Error; - Driver->Touch(EScan::Final); - } - -public: - static constexpr TStringBuf LogPrefix() { + +using namespace NActors; +using namespace NExportScan; +using namespace NTable; + +class TExportScan: private NActors::IActor, public NTable::IScan { + enum EStateBits { + ES_REGISTERED = 0, // Actor is registered + ES_INITIALIZED, // Seek(...) was called + ES_UPLOADER_READY, + ES_BUFFER_SENT, + ES_NO_MORE_DATA, + + ES_COUNT, + }; + + struct TStats { + ui64 Rows; + ui64 BytesRead; + ui64 BytesSent; + + TStats() + : Rows(0) + , BytesRead(0) + , BytesSent(0) + { + auto counters = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "store_to_yt"); + + MonRows = counters->GetCounter("Rows", true); + MonBytesRead = counters->GetCounter("BytesRead", true); + MonBytesSent = counters->GetCounter("BytesSent", true); + } + + void Aggr(ui64 rows, ui64 bytesRead, ui64 bytesSent) { + Rows += rows; + BytesRead += bytesRead; + BytesSent += bytesSent; + + *MonRows += rows; + *MonBytesRead += bytesRead; + *MonBytesSent += bytesSent; + } + + void Aggr(IBuffer const* buffer) { + Aggr(buffer->GetRows(), buffer->GetBytesRead(), buffer->GetBytesSent()); + } + + TString ToString() const { + return TStringBuilder() + << "Stats { " + << " Rows: " << Rows + << " BytesRead: " << BytesRead + << " BytesSent: " << BytesSent + << " }"; + } + + private: + NMonitoring::TDynamicCounters::TCounterPtr MonRows; + NMonitoring::TDynamicCounters::TCounterPtr MonBytesRead; + NMonitoring::TDynamicCounters::TCounterPtr MonBytesSent; + }; + + bool IsReady() const { + return State.Test(ES_REGISTERED) && State.Test(ES_INITIALIZED); + } + + void MaybeReady() { + if (IsReady()) { + Send(Uploader, new TEvExportScan::TEvReady()); + } + } + + EScan MaybeSendBuffer() { + const bool noMoreData = State.Test(ES_NO_MORE_DATA); + + if (!noMoreData && !Buffer->IsFilled()) { + return EScan::Feed; + } + + if (!State.Test(ES_UPLOADER_READY) || State.Test(ES_BUFFER_SENT)) { + Spent->Alter(false); + return EScan::Sleep; + } + + State.Set(ES_BUFFER_SENT); + Stats->Aggr(Buffer.Get()); + Send(Uploader, Buffer->PrepareEvent(noMoreData)); + + if (noMoreData) { + Spent->Alter(false); + return EScan::Sleep; + } + + return EScan::Feed; + } + + void Handle(TEvExportScan::TEvReset::TPtr&) { + Y_VERIFY(IsReady()); + + EXPORT_LOG_D("Handle TEvExportScan::TEvReset" + << ": self# " << SelfId()); + + Stats.Reset(new TStats); + State.Reset(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT).Reset(ES_NO_MORE_DATA); + Spent->Alter(true); + Driver->Touch(EScan::Reset); + } + + void Handle(TEvExportScan::TEvFeed::TPtr&) { + Y_VERIFY(IsReady()); + + EXPORT_LOG_D("Handle TEvExportScan::TEvFeed" + << ": self# " << SelfId()); + + State.Set(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT); + Spent->Alter(true); + if (EScan::Feed == MaybeSendBuffer()) { + Driver->Touch(EScan::Feed); + } + } + + void Handle(TEvExportScan::TEvFinish::TPtr& ev) { + Y_VERIFY(IsReady()); + + EXPORT_LOG_D("Handle TEvExportScan::TEvFinish" + << ": self# " << SelfId() + << ", msg# " << ev->Get()->ToString()); + + Success = ev->Get()->Success; + Error = ev->Get()->Error; + Driver->Touch(EScan::Final); + } + +public: + static constexpr TStringBuf LogPrefix() { return "scanner"sv; - } - - explicit TExportScan(std::function<IActor*()>&& createUploaderFn, IBuffer::TPtr buffer) - : IActor(static_cast<TReceiveFunc>(&TExportScan::StateWork), NKikimrServices::TActivity::EXPORT_SCAN_ACTOR) - , CreateUploaderFn(std::move(createUploaderFn)) - , Buffer(std::move(buffer)) - , Stats(new TStats) - , Driver(nullptr) - , Success(false) - { - } - - void Describe(IOutputStream& o) const noexcept override { - o << "ExportScan { " - << "Uploader: " << Uploader - << Stats->ToString() << " " - << "Success: " << Success - << "Error: " << Error - << " }"; - } - + } + + explicit TExportScan(std::function<IActor*()>&& createUploaderFn, IBuffer::TPtr buffer) + : IActor(static_cast<TReceiveFunc>(&TExportScan::StateWork), NKikimrServices::TActivity::EXPORT_SCAN_ACTOR) + , CreateUploaderFn(std::move(createUploaderFn)) + , Buffer(std::move(buffer)) + , Stats(new TStats) + , Driver(nullptr) + , Success(false) + { + } + + void Describe(IOutputStream& o) const noexcept override { + o << "ExportScan { " + << "Uploader: " << Uploader + << Stats->ToString() << " " + << "Success: " << Success + << "Error: " << Error + << " }"; + } + IScan::THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept override { TlsActivationContext->AsActorContext().RegisterWithSameMailbox(this); - - Driver = driver; - Scheme = std::move(scheme); - Spent = new TSpent(TAppData::TimeProvider.Get()); - Buffer->ColumnsOrder(Scheme->Tags()); - - return {EScan::Feed, {}}; - } - - void Registered(TActorSystem* sys, const TActorId&) override { - Uploader = sys->Register(CreateUploaderFn(), TMailboxType::HTSwap, AppData()->BatchPoolId); - - State.Set(ES_REGISTERED); - MaybeReady(); - } - - EScan Seek(TLead& lead, ui64) noexcept override { - lead.To(Scheme->Tags(), {}, ESeek::Lower); - Buffer->Clear(); - - State.Set(ES_INITIALIZED); - MaybeReady(); - - Spent->Alter(true); - return EScan::Feed; - } - - EScan Feed(TArrayRef<const TCell>, const TRow& row) noexcept override { - Buffer->Collect(row); - return MaybeSendBuffer(); - } - - EScan Exhausted() noexcept override { - State.Set(ES_NO_MORE_DATA); - return MaybeSendBuffer(); - } - + + Driver = driver; + Scheme = std::move(scheme); + Spent = new TSpent(TAppData::TimeProvider.Get()); + Buffer->ColumnsOrder(Scheme->Tags()); + + return {EScan::Feed, {}}; + } + + void Registered(TActorSystem* sys, const TActorId&) override { + Uploader = sys->Register(CreateUploaderFn(), TMailboxType::HTSwap, AppData()->BatchPoolId); + + State.Set(ES_REGISTERED); + MaybeReady(); + } + + EScan Seek(TLead& lead, ui64) noexcept override { + lead.To(Scheme->Tags(), {}, ESeek::Lower); + Buffer->Clear(); + + State.Set(ES_INITIALIZED); + MaybeReady(); + + Spent->Alter(true); + return EScan::Feed; + } + + EScan Feed(TArrayRef<const TCell>, const TRow& row) noexcept override { + Buffer->Collect(row); + return MaybeSendBuffer(); + } + + EScan Exhausted() noexcept override { + State.Set(ES_NO_MORE_DATA); + return MaybeSendBuffer(); + } + TAutoPtr<IDestructable> Finish(EAbort abort) noexcept override { - const bool success = (abort == EAbort::None) && Success; - - if (abort != EAbort::None) { - Error = "Aborted by scan host env"; - Send(Uploader, new TEvents::TEvPoisonPill()); - } - - PassAway(); - return new TExportScanProduct(success, Error, Stats->BytesRead, Stats->Rows); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExportScan::TEvReset, Handle); - hFunc(TEvExportScan::TEvFeed, Handle); - hFunc(TEvExportScan::TEvFinish, Handle); - } - } - -private: - std::function<IActor*()> CreateUploaderFn; - IBuffer::TPtr Buffer; - - TActorId Uploader; - THolder<TStats> Stats; - - IDriver* Driver; + const bool success = (abort == EAbort::None) && Success; + + if (abort != EAbort::None) { + Error = "Aborted by scan host env"; + Send(Uploader, new TEvents::TEvPoisonPill()); + } + + PassAway(); + return new TExportScanProduct(success, Error, Stats->BytesRead, Stats->Rows); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExportScan::TEvReset, Handle); + hFunc(TEvExportScan::TEvFeed, Handle); + hFunc(TEvExportScan::TEvFinish, Handle); + } + } + +private: + std::function<IActor*()> CreateUploaderFn; + IBuffer::TPtr Buffer; + + TActorId Uploader; + THolder<TStats> Stats; + + IDriver* Driver; TIntrusiveConstPtr<TScheme> Scheme; TAutoPtr<TSpent> Spent; - - TBitMap<EStateBits::ES_COUNT> State; - bool Success; - TString Error; - -}; // TExportScan - -NTable::IScan* CreateExportScan(IBuffer::TPtr buffer, std::function<IActor*()>&& createUploaderFn) { - return new TExportScan(std::move(createUploaderFn), std::move(buffer)); -} - + + TBitMap<EStateBits::ES_COUNT> State; + bool Success; + TString Error; + +}; // TExportScan + +NTable::IScan* CreateExportScan(IBuffer::TPtr buffer, std::function<IActor*()>&& createUploaderFn) { + return new TExportScan(std::move(createUploaderFn), std::move(buffer)); +} + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/export_scan.h b/ydb/core/tx/datashard/export_scan.h index adc79d3a8de..d077576a607 100644 --- a/ydb/core/tx/datashard/export_scan.h +++ b/ydb/core/tx/datashard/export_scan.h @@ -1,113 +1,113 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/tablet_flat/flat_scan_iface.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NKikimr { namespace NDataShard { - -struct TEvExportScan { - enum EEv { - EvReset = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvReady, - EvFeed, - EvBuffer, - EvFinish, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); - - struct TEvReset: public TEventLocal<TEvReset, EvReset> {}; - struct TEvReady: public TEventLocal<TEvReady, EvReady> {}; - struct TEvFeed: public TEventLocal<TEvFeed, EvFeed> {}; - - template <typename TBuffer> - struct TEvBuffer: public TEventLocal<TEvBuffer<TBuffer>, EvBuffer> { - TBuffer Buffer; - bool Last; - - TEvBuffer() = default; - - explicit TEvBuffer(TBuffer buffer, bool last) - : Buffer(std::move(buffer)) - , Last(last) - { - } - - TString ToString() const override { - return TStringBuilder() << this->ToStringHeader() << " {" - << " Last: " << Last - << " }"; - } - }; - - struct TEvFinish: public TEventLocal<TEvFinish, EvFinish> { - bool Success; - TString Error; - - TEvFinish() = default; - - explicit TEvFinish(bool success, const TString& error = TString()) - : Success(success) - , Error(error) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " {" - << " Success: " << Success - << " Error: " << Error - << " }"; - } - }; - -}; // TEvExportScan - + +struct TEvExportScan { + enum EEv { + EvReset = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvReady, + EvFeed, + EvBuffer, + EvFinish, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + struct TEvReset: public TEventLocal<TEvReset, EvReset> {}; + struct TEvReady: public TEventLocal<TEvReady, EvReady> {}; + struct TEvFeed: public TEventLocal<TEvFeed, EvFeed> {}; + + template <typename TBuffer> + struct TEvBuffer: public TEventLocal<TEvBuffer<TBuffer>, EvBuffer> { + TBuffer Buffer; + bool Last; + + TEvBuffer() = default; + + explicit TEvBuffer(TBuffer buffer, bool last) + : Buffer(std::move(buffer)) + , Last(last) + { + } + + TString ToString() const override { + return TStringBuilder() << this->ToStringHeader() << " {" + << " Last: " << Last + << " }"; + } + }; + + struct TEvFinish: public TEventLocal<TEvFinish, EvFinish> { + bool Success; + TString Error; + + TEvFinish() = default; + + explicit TEvFinish(bool success, const TString& error = TString()) + : Success(success) + , Error(error) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " Success: " << Success + << " Error: " << Error + << " }"; + } + }; + +}; // TEvExportScan + struct TExportScanProduct: public IDestructable { - bool Success; - TString Error; - ui64 BytesRead; - ui64 RowsRead; - - explicit TExportScanProduct(bool success, TString error, ui64 bytes, ui64 rows) - : Success(success) - , Error(std::move(error)) - , BytesRead(bytes) - , RowsRead(rows) - { - } - -}; // TExportScanProduct - -namespace NExportScan { - -class IBuffer { -public: - using TPtr = THolder<IBuffer>; - -public: - virtual ~IBuffer() = default; - - virtual void ColumnsOrder(const TVector<ui32>& tags) = 0; - virtual void Collect(const NTable::IScan::TRow& row) = 0; - virtual IEventBase* PrepareEvent(bool last) = 0; - virtual void Clear() = 0; - virtual bool IsFilled() const = 0; - virtual ui64 GetRows() const = 0; - virtual ui64 GetBytesRead() const = 0; - virtual ui64 GetBytesSent() const = 0; -}; - -} // NExportScan - -NTable::IScan* CreateExportScan(NExportScan::IBuffer::TPtr buffer, std::function<NActors::IActor*()>&& createUploaderFn); - + bool Success; + TString Error; + ui64 BytesRead; + ui64 RowsRead; + + explicit TExportScanProduct(bool success, TString error, ui64 bytes, ui64 rows) + : Success(success) + , Error(std::move(error)) + , BytesRead(bytes) + , RowsRead(rows) + { + } + +}; // TExportScanProduct + +namespace NExportScan { + +class IBuffer { +public: + using TPtr = THolder<IBuffer>; + +public: + virtual ~IBuffer() = default; + + virtual void ColumnsOrder(const TVector<ui32>& tags) = 0; + virtual void Collect(const NTable::IScan::TRow& row) = 0; + virtual IEventBase* PrepareEvent(bool last) = 0; + virtual void Clear() = 0; + virtual bool IsFilled() const = 0; + virtual ui64 GetRows() const = 0; + virtual ui64 GetBytesRead() const = 0; + virtual ui64 GetBytesSent() const = 0; +}; + +} // NExportScan + +NTable::IScan* CreateExportScan(NExportScan::IBuffer::TPtr buffer, std::function<NActors::IActor*()>&& createUploaderFn); + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/finish_propose_unit.cpp b/ydb/core/tx/datashard/finish_propose_unit.cpp index 59ce24e69ca..de0e9e4ad2e 100644 --- a/ydb/core/tx/datashard/finish_propose_unit.cpp +++ b/ydb/core/tx/datashard/finish_propose_unit.cpp @@ -115,8 +115,8 @@ void TFinishProposeUnit::Complete(TOperation::TPtr op, if (op->IsImmediate()) { Pipeline.RemoveCommittingOp(op); Pipeline.RemoveActiveOp(op); - - DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); + + DataShard.EnqueueChangeRecords(std::move(op->ChangeRecords())); } DataShard.SendRegistrationRequestTimeCast(ctx); diff --git a/ydb/core/tx/datashard/import_common.h b/ydb/core/tx/datashard/import_common.h index ddec66eb48c..a7cb38b18e7 100644 --- a/ydb/core/tx/datashard/import_common.h +++ b/ydb/core/tx/datashard/import_common.h @@ -1,133 +1,133 @@ -#pragma once - -#include "defs.h" -#include "datashard_user_table.h" - +#pragma once + +#include "defs.h" +#include "datashard_user_table.h" + #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/tablet_flat/util_basics.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -#if defined IMPORT_LOG_T || \ - defined IMPORT_LOG_D || \ - defined IMPORT_LOG_I || \ - defined IMPORT_LOG_N || \ - defined IMPORT_LOG_W || \ - defined IMPORT_LOG_E || \ - defined IMPORT_LOG_C -#error log macro redefinition -#endif - -#define IMPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) - -namespace NKikimr { + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +#if defined IMPORT_LOG_T || \ + defined IMPORT_LOG_D || \ + defined IMPORT_LOG_I || \ + defined IMPORT_LOG_N || \ + defined IMPORT_LOG_W || \ + defined IMPORT_LOG_E || \ + defined IMPORT_LOG_C +#error log macro redefinition +#endif + +#define IMPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) +#define IMPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) + +namespace NKikimr { namespace NDataShard { - + struct TImportJobProduct: public IDestructable { - bool Success; - TString Error; - ui64 BytesWritten; - ui64 RowsWritten; - - explicit TImportJobProduct(bool success, TString error, ui64 bytes, ui64 rows) - : Success(success) - , Error(std::move(error)) - , BytesWritten(bytes) - , RowsWritten(rows) - { - } - -}; // TImportJobProduct - -class TTableInfo { - using TColumnIdIndex = TMap<ui32, TUserTable::TUserColumn>; - using TColumnNameIndex = THashMap<TString, TColumnIdIndex::const_iterator>; - - static TColumnNameIndex MakeColumnNameIndex(TUserTable::TCPtr info) { - TColumnNameIndex result; - - auto it = info->Columns.begin(); - while (it != info->Columns.end()) { - Y_VERIFY(result.emplace(it->second.Name, it).second); - it++; - } - - return result; - } - -public: - explicit TTableInfo(ui64 id, TUserTable::TCPtr info) - : Id(id) - , Info(info) - , ColumnNameIndex(MakeColumnNameIndex(info)) - { - } - - ui64 GetId() const { - return Id; - } - - bool HasColumn(const TString& name) const { - return ColumnNameIndex.contains(name); - } - - NScheme::TTypeId GetColumnType(const TString& name) const { - auto it = ColumnNameIndex.find(name); - Y_VERIFY(it != ColumnNameIndex.end()); - return it->second->second.Type; - } - - const TVector<ui32>& GetKeyColumnIds() const { - return Info->KeyColumnIds; - } - - TVector<ui32> GetValueColumnIds(const TVector<TString>& order) const { - TVector<ui32> result; - for (const auto& name : order) { - auto it = ColumnNameIndex.find(name); - Y_VERIFY(it != ColumnNameIndex.end()); - - if (it->second->second.IsKey) { - continue; - } - - result.push_back(it->second->first); - } - - return result; - } - - ui32 KeyOrder(const TString& name) const { - auto it = ColumnNameIndex.find(name); - Y_VERIFY(it != ColumnNameIndex.end()); - - if (!it->second->second.IsKey) { - return Max<ui32>(); - } - - return FindIndex(Info->KeyColumnIds, it->second->first); - } - - bool IsMyKey(const TVector<TCell>& key) const { - return ComparePointAndRange(key, Info->GetTableRange(), Info->KeyColumnTypes, Info->KeyColumnTypes) == 0; - } - -private: - const ui64 Id; - TUserTable::TCPtr Info; - const TColumnNameIndex ColumnNameIndex; - -}; // TTableInfo - + bool Success; + TString Error; + ui64 BytesWritten; + ui64 RowsWritten; + + explicit TImportJobProduct(bool success, TString error, ui64 bytes, ui64 rows) + : Success(success) + , Error(std::move(error)) + , BytesWritten(bytes) + , RowsWritten(rows) + { + } + +}; // TImportJobProduct + +class TTableInfo { + using TColumnIdIndex = TMap<ui32, TUserTable::TUserColumn>; + using TColumnNameIndex = THashMap<TString, TColumnIdIndex::const_iterator>; + + static TColumnNameIndex MakeColumnNameIndex(TUserTable::TCPtr info) { + TColumnNameIndex result; + + auto it = info->Columns.begin(); + while (it != info->Columns.end()) { + Y_VERIFY(result.emplace(it->second.Name, it).second); + it++; + } + + return result; + } + +public: + explicit TTableInfo(ui64 id, TUserTable::TCPtr info) + : Id(id) + , Info(info) + , ColumnNameIndex(MakeColumnNameIndex(info)) + { + } + + ui64 GetId() const { + return Id; + } + + bool HasColumn(const TString& name) const { + return ColumnNameIndex.contains(name); + } + + NScheme::TTypeId GetColumnType(const TString& name) const { + auto it = ColumnNameIndex.find(name); + Y_VERIFY(it != ColumnNameIndex.end()); + return it->second->second.Type; + } + + const TVector<ui32>& GetKeyColumnIds() const { + return Info->KeyColumnIds; + } + + TVector<ui32> GetValueColumnIds(const TVector<TString>& order) const { + TVector<ui32> result; + for (const auto& name : order) { + auto it = ColumnNameIndex.find(name); + Y_VERIFY(it != ColumnNameIndex.end()); + + if (it->second->second.IsKey) { + continue; + } + + result.push_back(it->second->first); + } + + return result; + } + + ui32 KeyOrder(const TString& name) const { + auto it = ColumnNameIndex.find(name); + Y_VERIFY(it != ColumnNameIndex.end()); + + if (!it->second->second.IsKey) { + return Max<ui32>(); + } + + return FindIndex(Info->KeyColumnIds, it->second->first); + } + + bool IsMyKey(const TVector<TCell>& key) const { + return ComparePointAndRange(key, Info->GetTableRange(), Info->KeyColumnTypes, Info->KeyColumnTypes) == 0; + } + +private: + const ui64 Id; + TUserTable::TCPtr Info; + const TColumnNameIndex ColumnNameIndex; + +}; // TTableInfo + } // NDataShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index ac8d0d79096..9ea7fbab5c0 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -1,10 +1,10 @@ -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "datashard_impl.h" -#include "import_common.h" -#include "import_s3.h" -#include "s3_common.h" - +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "datashard_impl.h" +#include "import_common.h" +#include "import_s3.h" +#include "s3_common.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/protos/services.pb.h> @@ -12,537 +12,537 @@ #include <ydb/core/wrappers/s3_wrapper.h> #include <ydb/core/io_formats/csv.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/memory/pool.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/memory/pool.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NDataShard { - -using namespace NResourceBroker; -using namespace NWrappers; - -using namespace Aws::S3; -using namespace Aws; - + +using namespace NResourceBroker; +using namespace NWrappers; + +using namespace Aws::S3; +using namespace Aws; + class TS3Downloader: public TActorBootstrapped<TS3Downloader>, private TS3User { - class TReadController { - static constexpr ui64 SumWithSaturation(ui64 a, ui64 b) { - return Max<ui64>() - a < b ? Max<ui64>() : a + b; - } - - public: - explicit TReadController(ui32 rangeSize) - : RangeSize(rangeSize) - { - } - - std::pair<ui64, ui64> NextRange(ui64 contentLength, ui64 processedBytes) const { - Y_VERIFY(contentLength > 0); - Y_VERIFY(processedBytes < contentLength); - - const ui64 start = processedBytes + (Buffer.size() - Pos); - const ui64 end = Min(SumWithSaturation(start, RangeSize), contentLength) - 1; - return std::make_pair(start, end); - } - - bool Feed(TString&& portion, TStringBuf& buf) { - if (Buffer && Pos) { - Buffer.remove(0, Pos); - } - - if (!Buffer) { - Buffer = std::move(portion); - } else { - Buffer.append(portion); - } - - Pos = Buffer.rfind('\n'); - if (TString::npos == Pos) { - return false; - } - - Pos += 1; // for '\n' - buf = TStringBuf(Buffer.data(), Pos); - - return true; - } - - private: - const ui32 RangeSize; - - TString Buffer; - ui64 Pos = 0; - - }; // TReadController - - class TUploadRowsRequestBuilder { - public: + class TReadController { + static constexpr ui64 SumWithSaturation(ui64 a, ui64 b) { + return Max<ui64>() - a < b ? Max<ui64>() : a + b; + } + + public: + explicit TReadController(ui32 rangeSize) + : RangeSize(rangeSize) + { + } + + std::pair<ui64, ui64> NextRange(ui64 contentLength, ui64 processedBytes) const { + Y_VERIFY(contentLength > 0); + Y_VERIFY(processedBytes < contentLength); + + const ui64 start = processedBytes + (Buffer.size() - Pos); + const ui64 end = Min(SumWithSaturation(start, RangeSize), contentLength) - 1; + return std::make_pair(start, end); + } + + bool Feed(TString&& portion, TStringBuf& buf) { + if (Buffer && Pos) { + Buffer.remove(0, Pos); + } + + if (!Buffer) { + Buffer = std::move(portion); + } else { + Buffer.append(portion); + } + + Pos = Buffer.rfind('\n'); + if (TString::npos == Pos) { + return false; + } + + Pos += 1; // for '\n' + buf = TStringBuf(Buffer.data(), Pos); + + return true; + } + + private: + const ui32 RangeSize; + + TString Buffer; + ui64 Pos = 0; + + }; // TReadController + + class TUploadRowsRequestBuilder { + public: void New(const TTableInfo& tableInfo, const NKikimrSchemeOp::TTableDescription& scheme) { - Request.Reset(new TEvDataShard::TEvUnsafeUploadRowsRequest()); - Request->Record.SetTableId(tableInfo.GetId()); - - TVector<TString> columnNames; - for (const auto& column : scheme.GetColumns()) { - columnNames.push_back(column.GetName()); - } - - auto& rowScheme = *Request->Record.MutableRowScheme(); - for (ui32 id : tableInfo.GetKeyColumnIds()) { - rowScheme.AddKeyColumnIds(id); - } - for (ui32 id : tableInfo.GetValueColumnIds(columnNames)) { - rowScheme.AddValueColumnIds(id); - } - } - - void AddRow(const TVector<TCell>& keys, const TVector<TCell>& values) { - Y_VERIFY(Request); - auto& row = *Request->Record.AddRows(); - row.SetKeyColumns(TSerializedCellVec::Serialize(keys)); - row.SetValueColumns(TSerializedCellVec::Serialize(values)); - } - - const NKikimrTxDataShard::TEvUploadRowsRequest& GetRecord() const { - Y_VERIFY(Request); - return Request->Record; - } - - THolder<TEvDataShard::TEvUnsafeUploadRowsRequest> Build() { - Y_VERIFY(Request); - return std::move(Request); - } - - private: - THolder<TEvDataShard::TEvUnsafeUploadRowsRequest> Request; - - }; // TUploadRowsRequestBuilder - - void AllocateResource() { - IMPORT_LOG_D("AllocateResource" - << ": self# " << SelfId()); - - const auto* appData = AppData(); - Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvSubmitTask( - TStringBuilder() << "Restore { " << TxId << ":" << DataShard << " }", - {{ 1, 0 }}, - appData->DataShardConfig.GetRestoreTaskName(), - appData->DataShardConfig.GetRestoreTaskPriority(), - nullptr - )); - - Become(&TThis::StateAllocateResource); - } - - void Handle(TEvResourceBroker::TEvResourceAllocated::TPtr& ev) { - IMPORT_LOG_D("Handle TEvResourceBroker::TEvResourceAllocated" - << ": self# " << SelfId() - << ", taskId# " << ev->Get()->TaskId); - - TaskId = ev->Get()->TaskId; - Restart(); - } - - void Restart() { - IMPORT_LOG_D("Restart" - << ": self# " << SelfId() - << ", attempt# " << Attempt); - - if (const TActorId client = std::exchange(Client, TActorId())) { - Send(client, new TEvents::TEvPoisonPill()); - } - - Client = RegisterWithSameMailbox(CreateS3Wrapper(Settings.Credentials, Settings.Config)); - - HeadObject(Settings.DataKey); - Become(&TThis::StateWork); - } - - void HeadObject(const TString& key) { - IMPORT_LOG_D("HeadObject" - << ": self# " << SelfId() - << ", key# " << key); - - auto request = Model::HeadObjectRequest() - .WithBucket(Settings.Bucket) - .WithKey(key); - - Send(Client, new TEvS3Wrapper::TEvHeadObjectRequest(request)); - } - - void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { - IMPORT_LOG_D("GetObject" - << ": self# " << SelfId() - << ", key# " << key - << ", range# " << range.first << "-" << range.second); - - auto request = Model::GetObjectRequest() - .WithBucket(Settings.Bucket) - .WithKey(key) - .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); - - Send(Client, new TEvS3Wrapper::TEvGetObjectRequest(request)); - } - - void Handle(TEvS3Wrapper::TEvHeadObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - IMPORT_LOG_D("Handle TEvS3Wrapper::TEvHeadObjectResponse" - << ": self# " << SelfId() - << ", result# " << result); - + Request.Reset(new TEvDataShard::TEvUnsafeUploadRowsRequest()); + Request->Record.SetTableId(tableInfo.GetId()); + + TVector<TString> columnNames; + for (const auto& column : scheme.GetColumns()) { + columnNames.push_back(column.GetName()); + } + + auto& rowScheme = *Request->Record.MutableRowScheme(); + for (ui32 id : tableInfo.GetKeyColumnIds()) { + rowScheme.AddKeyColumnIds(id); + } + for (ui32 id : tableInfo.GetValueColumnIds(columnNames)) { + rowScheme.AddValueColumnIds(id); + } + } + + void AddRow(const TVector<TCell>& keys, const TVector<TCell>& values) { + Y_VERIFY(Request); + auto& row = *Request->Record.AddRows(); + row.SetKeyColumns(TSerializedCellVec::Serialize(keys)); + row.SetValueColumns(TSerializedCellVec::Serialize(values)); + } + + const NKikimrTxDataShard::TEvUploadRowsRequest& GetRecord() const { + Y_VERIFY(Request); + return Request->Record; + } + + THolder<TEvDataShard::TEvUnsafeUploadRowsRequest> Build() { + Y_VERIFY(Request); + return std::move(Request); + } + + private: + THolder<TEvDataShard::TEvUnsafeUploadRowsRequest> Request; + + }; // TUploadRowsRequestBuilder + + void AllocateResource() { + IMPORT_LOG_D("AllocateResource" + << ": self# " << SelfId()); + + const auto* appData = AppData(); + Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvSubmitTask( + TStringBuilder() << "Restore { " << TxId << ":" << DataShard << " }", + {{ 1, 0 }}, + appData->DataShardConfig.GetRestoreTaskName(), + appData->DataShardConfig.GetRestoreTaskPriority(), + nullptr + )); + + Become(&TThis::StateAllocateResource); + } + + void Handle(TEvResourceBroker::TEvResourceAllocated::TPtr& ev) { + IMPORT_LOG_D("Handle TEvResourceBroker::TEvResourceAllocated" + << ": self# " << SelfId() + << ", taskId# " << ev->Get()->TaskId); + + TaskId = ev->Get()->TaskId; + Restart(); + } + + void Restart() { + IMPORT_LOG_D("Restart" + << ": self# " << SelfId() + << ", attempt# " << Attempt); + + if (const TActorId client = std::exchange(Client, TActorId())) { + Send(client, new TEvents::TEvPoisonPill()); + } + + Client = RegisterWithSameMailbox(CreateS3Wrapper(Settings.Credentials, Settings.Config)); + + HeadObject(Settings.DataKey); + Become(&TThis::StateWork); + } + + void HeadObject(const TString& key) { + IMPORT_LOG_D("HeadObject" + << ": self# " << SelfId() + << ", key# " << key); + + auto request = Model::HeadObjectRequest() + .WithBucket(Settings.Bucket) + .WithKey(key); + + Send(Client, new TEvS3Wrapper::TEvHeadObjectRequest(request)); + } + + void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { + IMPORT_LOG_D("GetObject" + << ": self# " << SelfId() + << ", key# " << key + << ", range# " << range.first << "-" << range.second); + + auto request = Model::GetObjectRequest() + .WithBucket(Settings.Bucket) + .WithKey(key) + .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); + + Send(Client, new TEvS3Wrapper::TEvGetObjectRequest(request)); + } + + void Handle(TEvS3Wrapper::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + IMPORT_LOG_D("Handle TEvS3Wrapper::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + if (!CheckResult(result, TStringBuf("HeadObject"))) { - return; - } - - ETag = result.GetResult().GetETag(); - ContentLength = result.GetResult().GetContentLength(); - - Send(DataShard, new TEvDataShard::TEvGetS3DownloadInfo(SelfId(), TxId)); - } - - void Handle(TEvDataShard::TEvS3DownloadInfo::TPtr& ev) { - const auto& info = ev->Get()->Info; - - IMPORT_LOG_D("Handle TEvDataShard::TEvS3DownloadInfo" - << ": self# " << SelfId() - << ", info# " << info.ToString()); - - if (!info.DataETag) { - Send(DataShard, new TEvDataShard::TEvStoreS3DownloadInfo(SelfId(), - TxId, ETag, ProcessedBytes, WrittenBytes, WrittenRows)); - return; - } - + return; + } + + ETag = result.GetResult().GetETag(); + ContentLength = result.GetResult().GetContentLength(); + + Send(DataShard, new TEvDataShard::TEvGetS3DownloadInfo(SelfId(), TxId)); + } + + void Handle(TEvDataShard::TEvS3DownloadInfo::TPtr& ev) { + const auto& info = ev->Get()->Info; + + IMPORT_LOG_D("Handle TEvDataShard::TEvS3DownloadInfo" + << ": self# " << SelfId() + << ", info# " << info.ToString()); + + if (!info.DataETag) { + Send(DataShard, new TEvDataShard::TEvStoreS3DownloadInfo(SelfId(), + TxId, ETag, ProcessedBytes, WrittenBytes, WrittenRows)); + return; + } + if (!CheckETag(*info.DataETag, ETag, TStringBuf("DownloadInfo"))) { - return; - } - - ProcessedBytes = info.ProcessedBytes; - WrittenBytes = info.WrittenBytes; - WrittenRows = info.WrittenRows; - - if (!ContentLength || ProcessedBytes >= ContentLength) { - return Finish(); - } - - GetObject(Settings.DataKey, Reader.NextRange(ContentLength, ProcessedBytes)); - } - - void Handle(TEvS3Wrapper::TEvGetObjectResponse::TPtr& ev) { - auto& msg = *ev->Get(); - const auto& result = msg.Result; - - IMPORT_LOG_D("Handle TEvS3Wrapper::TEvGetObjectResponse" - << ": self# " << SelfId() - << ", result# " << result); - - const TStringBuf marker = "GetObject"; - - if (!CheckResult(result, marker)) { - return; - } - - if (!CheckETag(ETag, result.GetResult().c_str(), marker)) { - return; - } - - IMPORT_LOG_T("Content size" - << ": self# " << SelfId() - << ", processed-bytes# " << ProcessedBytes - << ", content-length# " << ContentLength - << ", body-size# " << msg.Body.size()); - - if (!Reader.Feed(std::move(msg.Body), Buffer)) { - return Finish(false, "Cannot find new line symbol in data"); - } - - ProcessedBytes += Buffer.size(); - RequestBuilder.New(TableInfo, Scheme); - - TMemoryPool pool(256); - while (ProcessData(pool)); - } - - bool ProcessData(TMemoryPool& pool) { - pool.Clear(); - - TStringBuf line = Buffer.NextTok('\n'); - const TStringBuf origLine = line; - - if (!line) { - if (Buffer) { - return true; // skip empty line - } - - const auto& record = RequestBuilder.GetRecord(); - WrittenRows += record.RowsSize(); - - IMPORT_LOG_D("Upload rows" - << ": self# " << SelfId() - << ", count# " << record.RowsSize() - << ", size# " << record.ByteSizeLong()); - - Send(DataShard, RequestBuilder.Build().Release()); - return false; - } - + return; + } + + ProcessedBytes = info.ProcessedBytes; + WrittenBytes = info.WrittenBytes; + WrittenRows = info.WrittenRows; + + if (!ContentLength || ProcessedBytes >= ContentLength) { + return Finish(); + } + + GetObject(Settings.DataKey, Reader.NextRange(ContentLength, ProcessedBytes)); + } + + void Handle(TEvS3Wrapper::TEvGetObjectResponse::TPtr& ev) { + auto& msg = *ev->Get(); + const auto& result = msg.Result; + + IMPORT_LOG_D("Handle TEvS3Wrapper::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + const TStringBuf marker = "GetObject"; + + if (!CheckResult(result, marker)) { + return; + } + + if (!CheckETag(ETag, result.GetResult().c_str(), marker)) { + return; + } + + IMPORT_LOG_T("Content size" + << ": self# " << SelfId() + << ", processed-bytes# " << ProcessedBytes + << ", content-length# " << ContentLength + << ", body-size# " << msg.Body.size()); + + if (!Reader.Feed(std::move(msg.Body), Buffer)) { + return Finish(false, "Cannot find new line symbol in data"); + } + + ProcessedBytes += Buffer.size(); + RequestBuilder.New(TableInfo, Scheme); + + TMemoryPool pool(256); + while (ProcessData(pool)); + } + + bool ProcessData(TMemoryPool& pool) { + pool.Clear(); + + TStringBuf line = Buffer.NextTok('\n'); + const TStringBuf origLine = line; + + if (!line) { + if (Buffer) { + return true; // skip empty line + } + + const auto& record = RequestBuilder.GetRecord(); + WrittenRows += record.RowsSize(); + + IMPORT_LOG_D("Upload rows" + << ": self# " << SelfId() + << ", count# " << record.RowsSize() + << ", size# " << record.ByteSizeLong()); + + Send(DataShard, RequestBuilder.Build().Release()); + return false; + } + std::vector<std::pair<i32, ui32>> columnOrderTypes; // {keyOrder, PType} columnOrderTypes.reserve(Scheme.GetColumns().size()); - - for (const auto& column : Scheme.GetColumns()) { + + for (const auto& column : Scheme.GetColumns()) { columnOrderTypes.emplace_back(TableInfo.KeyOrder(column.GetName()), column.GetTypeId()); } - + TVector<TCell> keys; TVector<TCell> values; TString strError; - + if (!NFormats::TYdbDump::ParseLine(line, columnOrderTypes, pool, keys, values, strError, WrittenBytes)) { Finish(false, TStringBuilder() << strError << " on line: " << origLine); return false; - } - - Y_VERIFY(!keys.empty()); - if (!TableInfo.IsMyKey(keys) /* TODO: maybe skip */) { - Finish(false, TStringBuilder() << "Key is out of range on line: " << origLine); - return false; - } - - RequestBuilder.AddRow(keys, values); - return true; - } - - void Handle(TEvDataShard::TEvUnsafeUploadRowsResponse::TPtr& ev) { - const auto& record = ev->Get()->Record; - - IMPORT_LOG_D("Handle TEvDataShard::TEvUnsafeUploadRowsResponse" - << ": self# " << SelfId() - << ", record# " << record.ShortDebugString()); - - switch (record.GetStatus()) { - case NKikimrTxDataShard::TError::OK: - break; - - case NKikimrTxDataShard::TError::SCHEME_ERROR: - case NKikimrTxDataShard::TError::BAD_ARGUMENT: - return Finish(false, record.GetErrorDescription()); - - default: - return RetryOrFinish(record.GetErrorDescription()); - }; - - Send(DataShard, new TEvDataShard::TEvStoreS3DownloadInfo(SelfId(), - TxId, ETag, ProcessedBytes, WrittenBytes, WrittenRows)); - } - - template <typename TResult> - bool CheckResult(const TResult& result, const TStringBuf marker) { - if (result.IsSuccess()) { - return true; - } - - IMPORT_LOG_E("Error at '" << marker << "'" - << ": self# " << SelfId() - << ", error# " << result); - RetryOrFinish(result.GetError().GetMessage().c_str()); - - return false; - } - - bool CheckETag(const TString& expected, const TString& got, const TStringBuf marker) { - if (expected == got) { - return true; - } - - const TString error = TStringBuilder() << "ETag mismatch at '" << marker << "'" - << ": self# " << SelfId() - << ", expected# " << expected - << ", got# " << got; - - IMPORT_LOG_E(error); - Finish(false, error); - - return false; - } - - bool CheckScheme() { - auto finish = [this](const TString& error) -> bool { - IMPORT_LOG_E(error); - Finish(false, error); - - return false; - }; - - for (const auto& column : Scheme.GetColumns()) { - if (!TableInfo.HasColumn(column.GetName())) { - return finish(TStringBuilder() << "Scheme mismatch: cannot find column" - << ": name# " << column.GetName()); - } - - const NScheme::TTypeId type = TableInfo.GetColumnType(column.GetName()); - if (type != static_cast<NScheme::TTypeId>(column.GetTypeId())) { - return finish(TStringBuilder() << "Scheme mismatch: column type mismatch" - << ": name# " << column.GetName() - << ", expected# " << type - << ", got# " << static_cast<NScheme::TTypeId>(column.GetTypeId())); - } - } - - if (TableInfo.GetKeyColumnIds().size() != (ui32)Scheme.KeyColumnNamesSize()) { - return finish(TStringBuilder() << "Scheme mismatch: key column count mismatch" - << ": expected# " << TableInfo.GetKeyColumnIds().size() - << ", got# " << Scheme.KeyColumnNamesSize()); - } - - for (ui32 i = 0; i < (ui32)Scheme.KeyColumnNamesSize(); ++i) { - const auto& name = Scheme.GetKeyColumnNames(i); - const ui32 keyOrder = TableInfo.KeyOrder(name); - - if (keyOrder != i) { - return finish(TStringBuilder() << "Scheme mismatch: key order mismatch" - << ": name# " << name - << ", expected# " << keyOrder - << ", got# " << i); - } - } - - return true; - } - - void RetryOrFinish(const TString& error) { - if (Attempt++ < Retries) { - Delay = Min(Delay * Attempt, MaxDelay); - const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); - - Schedule(Delay + random, new TEvents::TEvWakeup()); - } else { - Finish(false, error); - } - } - - void Finish(bool success = true, const TString& error = TString()) { - IMPORT_LOG_I("Finish" - << ": self# " << SelfId() - << ", success# " << success - << ", error# " << error - << ", writtenBytes# " << WrittenBytes - << ", writtenRows# " << WrittenRows); - + } + + Y_VERIFY(!keys.empty()); + if (!TableInfo.IsMyKey(keys) /* TODO: maybe skip */) { + Finish(false, TStringBuilder() << "Key is out of range on line: " << origLine); + return false; + } + + RequestBuilder.AddRow(keys, values); + return true; + } + + void Handle(TEvDataShard::TEvUnsafeUploadRowsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + IMPORT_LOG_D("Handle TEvDataShard::TEvUnsafeUploadRowsResponse" + << ": self# " << SelfId() + << ", record# " << record.ShortDebugString()); + + switch (record.GetStatus()) { + case NKikimrTxDataShard::TError::OK: + break; + + case NKikimrTxDataShard::TError::SCHEME_ERROR: + case NKikimrTxDataShard::TError::BAD_ARGUMENT: + return Finish(false, record.GetErrorDescription()); + + default: + return RetryOrFinish(record.GetErrorDescription()); + }; + + Send(DataShard, new TEvDataShard::TEvStoreS3DownloadInfo(SelfId(), + TxId, ETag, ProcessedBytes, WrittenBytes, WrittenRows)); + } + + template <typename TResult> + bool CheckResult(const TResult& result, const TStringBuf marker) { + if (result.IsSuccess()) { + return true; + } + + IMPORT_LOG_E("Error at '" << marker << "'" + << ": self# " << SelfId() + << ", error# " << result); + RetryOrFinish(result.GetError().GetMessage().c_str()); + + return false; + } + + bool CheckETag(const TString& expected, const TString& got, const TStringBuf marker) { + if (expected == got) { + return true; + } + + const TString error = TStringBuilder() << "ETag mismatch at '" << marker << "'" + << ": self# " << SelfId() + << ", expected# " << expected + << ", got# " << got; + + IMPORT_LOG_E(error); + Finish(false, error); + + return false; + } + + bool CheckScheme() { + auto finish = [this](const TString& error) -> bool { + IMPORT_LOG_E(error); + Finish(false, error); + + return false; + }; + + for (const auto& column : Scheme.GetColumns()) { + if (!TableInfo.HasColumn(column.GetName())) { + return finish(TStringBuilder() << "Scheme mismatch: cannot find column" + << ": name# " << column.GetName()); + } + + const NScheme::TTypeId type = TableInfo.GetColumnType(column.GetName()); + if (type != static_cast<NScheme::TTypeId>(column.GetTypeId())) { + return finish(TStringBuilder() << "Scheme mismatch: column type mismatch" + << ": name# " << column.GetName() + << ", expected# " << type + << ", got# " << static_cast<NScheme::TTypeId>(column.GetTypeId())); + } + } + + if (TableInfo.GetKeyColumnIds().size() != (ui32)Scheme.KeyColumnNamesSize()) { + return finish(TStringBuilder() << "Scheme mismatch: key column count mismatch" + << ": expected# " << TableInfo.GetKeyColumnIds().size() + << ", got# " << Scheme.KeyColumnNamesSize()); + } + + for (ui32 i = 0; i < (ui32)Scheme.KeyColumnNamesSize(); ++i) { + const auto& name = Scheme.GetKeyColumnNames(i); + const ui32 keyOrder = TableInfo.KeyOrder(name); + + if (keyOrder != i) { + return finish(TStringBuilder() << "Scheme mismatch: key order mismatch" + << ": name# " << name + << ", expected# " << keyOrder + << ", got# " << i); + } + } + + return true; + } + + void RetryOrFinish(const TString& error) { + if (Attempt++ < Retries) { + Delay = Min(Delay * Attempt, MaxDelay); + const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); + + Schedule(Delay + random, new TEvents::TEvWakeup()); + } else { + Finish(false, error); + } + } + + void Finish(bool success = true, const TString& error = TString()) { + IMPORT_LOG_I("Finish" + << ": self# " << SelfId() + << ", success# " << success + << ", error# " << error + << ", writtenBytes# " << WrittenBytes + << ", writtenRows# " << WrittenRows); + TAutoPtr<IDestructable> prod = new TImportJobProduct(success, error, WrittenBytes, WrittenRows); Send(DataShard, new TDataShard::TEvPrivate::TEvAsyncJobComplete(prod), 0, TxId); - - Y_VERIFY(TaskId); - Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvFinishTask(TaskId)); - - PassAway(); - } - - void NotifyDied() { - Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvNotifyActorDied()); - PassAway(); - } - - void PassAway() override { - if (Client) { - Send(Client, new TEvents::TEvPoisonPill()); - } - - TActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::IMPORT_S3_DOWNLOADER_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { - return "s3"; - } - + + Y_VERIFY(TaskId); + Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvFinishTask(TaskId)); + + PassAway(); + } + + void NotifyDied() { + Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvNotifyActorDied()); + PassAway(); + } + + void PassAway() override { + if (Client) { + Send(Client, new TEvents::TEvPoisonPill()); + } + + TActor::PassAway(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::IMPORT_S3_DOWNLOADER_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { + return "s3"; + } + explicit TS3Downloader(const TActorId& dataShard, ui64 txId, const NKikimrSchemeOp::TRestoreTask& task, const TTableInfo& tableInfo) - : DataShard(dataShard) - , TxId(txId) - , Settings(TS3Settings::FromRestoreTask(task)) - , TableInfo(tableInfo) - , Scheme(task.GetTableDescription()) - , Retries(task.GetNumberOfRetries()) - , Reader(task.GetS3Settings().GetLimits().GetReadBatchSize()) - { - } - - void Bootstrap() { - IMPORT_LOG_D("Bootstrap" - << ": self# " << SelfId() - << ", attempt# " << Attempt); - - if (!CheckScheme()) { - return; - } - - AllocateResource(); - } - - STATEFN(StateAllocateResource) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvResourceBroker::TEvResourceAllocated, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, NotifyDied); - } - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvS3Wrapper::TEvHeadObjectResponse, Handle); - hFunc(TEvS3Wrapper::TEvGetObjectResponse, Handle); - - hFunc(TEvDataShard::TEvS3DownloadInfo, Handle); - hFunc(TEvDataShard::TEvUnsafeUploadRowsResponse, Handle); - - cFunc(TEvents::TEvWakeup::EventType, Restart); - cFunc(TEvents::TEvPoisonPill::EventType, NotifyDied); - } - } - -private: - const TActorId DataShard; - const ui64 TxId; - const TS3Settings Settings; - const TTableInfo TableInfo; + : DataShard(dataShard) + , TxId(txId) + , Settings(TS3Settings::FromRestoreTask(task)) + , TableInfo(tableInfo) + , Scheme(task.GetTableDescription()) + , Retries(task.GetNumberOfRetries()) + , Reader(task.GetS3Settings().GetLimits().GetReadBatchSize()) + { + } + + void Bootstrap() { + IMPORT_LOG_D("Bootstrap" + << ": self# " << SelfId() + << ", attempt# " << Attempt); + + if (!CheckScheme()) { + return; + } + + AllocateResource(); + } + + STATEFN(StateAllocateResource) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvResourceBroker::TEvResourceAllocated, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, NotifyDied); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvS3Wrapper::TEvHeadObjectResponse, Handle); + hFunc(TEvS3Wrapper::TEvGetObjectResponse, Handle); + + hFunc(TEvDataShard::TEvS3DownloadInfo, Handle); + hFunc(TEvDataShard::TEvUnsafeUploadRowsResponse, Handle); + + cFunc(TEvents::TEvWakeup::EventType, Restart); + cFunc(TEvents::TEvPoisonPill::EventType, NotifyDied); + } + } + +private: + const TActorId DataShard; + const ui64 TxId; + const TS3Settings Settings; + const TTableInfo TableInfo; const NKikimrSchemeOp::TTableDescription Scheme; - - const ui32 Retries; - ui32 Attempt = 0; - - TDuration Delay = TDuration::Minutes(1); - static constexpr TDuration MaxDelay = TDuration::Minutes(10); - - ui64 TaskId = 0; - TActorId Client; - - TString ETag; - ui64 ContentLength = 0; - ui64 ProcessedBytes = 0; - ui64 WrittenBytes = 0; - ui64 WrittenRows = 0; - - TReadController Reader; - TStringBuf Buffer; - TUploadRowsRequestBuilder RequestBuilder; - -}; // TS3Downloader - + + const ui32 Retries; + ui32 Attempt = 0; + + TDuration Delay = TDuration::Minutes(1); + static constexpr TDuration MaxDelay = TDuration::Minutes(10); + + ui64 TaskId = 0; + TActorId Client; + + TString ETag; + ui64 ContentLength = 0; + ui64 ProcessedBytes = 0; + ui64 WrittenBytes = 0; + ui64 WrittenRows = 0; + + TReadController Reader; + TStringBuf Buffer; + TUploadRowsRequestBuilder RequestBuilder; + +}; // TS3Downloader + IActor* CreateS3Downloader(const TActorId& dataShard, ui64 txId, const NKikimrSchemeOp::TRestoreTask& task, const TTableInfo& info) { - return new TS3Downloader(dataShard, txId, task, info); -} - + return new TS3Downloader(dataShard, txId, task, info); +} + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/import_s3.h b/ydb/core/tx/datashard/import_s3.h index 462aeee33ad..f879693de8d 100644 --- a/ydb/core/tx/datashard/import_s3.h +++ b/ydb/core/tx/datashard/import_s3.h @@ -1,21 +1,21 @@ -#pragma once - -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "defs.h" - +#pragma once + +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "defs.h" + namespace NKikimrSchemeOp { - class TRestoreTask; -} - -namespace NKikimr { + class TRestoreTask; +} + +namespace NKikimr { namespace NDataShard { - -class TTableInfo; - + +class TTableInfo; + IActor* CreateS3Downloader(const TActorId& dataShard, ui64 txId, const NKikimrSchemeOp::TRestoreTask& task, const TTableInfo& info); - + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/initiate_build_index_unit.cpp b/ydb/core/tx/datashard/initiate_build_index_unit.cpp index 5e52dced17b..f52034aeea6 100644 --- a/ydb/core/tx/datashard/initiate_build_index_unit.cpp +++ b/ydb/core/tx/datashard/initiate_build_index_unit.cpp @@ -6,8 +6,8 @@ namespace NKikimr { namespace NDataShard { class TInitiateBuildIndexUnit : public TExecutionUnit { - THolder<TEvChangeExchange::TEvAddSender> AddSender; - + THolder<TEvChangeExchange::TEvAddSender> AddSender; + public: TInitiateBuildIndexUnit(TDataShard& dataShard, TPipeline& pipeline) : TExecutionUnit(EExecutionUnitKind::InitiateBuildIndex, false, dataShard, pipeline) @@ -33,23 +33,23 @@ public: auto pathId = TPathId(params.GetPathId().GetOwnerId(), params.GetPathId().GetLocalId()); Y_VERIFY(pathId.OwnerId == DataShard.GetPathOwnerId()); - TUserTable::TPtr tableInfo; - if (params.HasIndexDescription()) { - const auto& indexDesc = params.GetIndexDescription(); - + TUserTable::TPtr tableInfo; + if (params.HasIndexDescription()) { + const auto& indexDesc = params.GetIndexDescription(); + if (indexDesc.GetType() == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync) { - auto indexPathId = TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()); - AddSender.Reset(new TEvChangeExchange::TEvAddSender( - pathId, TEvChangeExchange::ESenderType::AsyncIndex, indexPathId - )); - } - - tableInfo = DataShard.AlterTableAddIndex(ctx, txc, pathId, params.GetTableSchemaVersion(), indexDesc); - } else { - tableInfo = DataShard.AlterTableSchemaVersion(ctx, txc, pathId, params.GetTableSchemaVersion()); - } - - Y_VERIFY(tableInfo); + auto indexPathId = TPathId(indexDesc.GetPathOwnerId(), indexDesc.GetLocalPathId()); + AddSender.Reset(new TEvChangeExchange::TEvAddSender( + pathId, TEvChangeExchange::ESenderType::AsyncIndex, indexPathId + )); + } + + tableInfo = DataShard.AlterTableAddIndex(ctx, txc, pathId, params.GetTableSchemaVersion(), indexDesc); + } else { + tableInfo = DataShard.AlterTableSchemaVersion(ctx, txc, pathId, params.GetTableSchemaVersion()); + } + + Y_VERIFY(tableInfo); DataShard.AddUserTable(pathId, tableInfo); ui64 step = tx->GetStep(); @@ -67,16 +67,16 @@ public: op->Result()->SetStepOrderId(op->GetStepOrder().ToPair()); if (added) { - return EExecutionStatus::DelayCompleteNoMoreRestarts; + return EExecutionStatus::DelayCompleteNoMoreRestarts; } else { - return EExecutionStatus::DelayComplete; + return EExecutionStatus::DelayComplete; } } - void Complete(TOperation::TPtr, const TActorContext& ctx) override { - if (AddSender) { - ctx.Send(DataShard.GetChangeSender(), AddSender.Release()); - } + void Complete(TOperation::TPtr, const TActorContext& ctx) override { + if (AddSender) { + ctx.Send(DataShard.GetChangeSender(), AddSender.Release()); + } } }; diff --git a/ydb/core/tx/datashard/operation.h b/ydb/core/tx/datashard/operation.h index 2d420e3f2c2..0f4b713e91c 100644 --- a/ydb/core/tx/datashard/operation.h +++ b/ydb/core/tx/datashard/operation.h @@ -5,7 +5,7 @@ #include "datashard_locks.h" #include "datashard_outreadset.h" #include "datashard_snapshots.h" -#include "change_exchange.h" +#include "change_exchange.h" #include "execution_unit_kind.h" #include <ydb/core/engine/mkql_engine_flat.h> @@ -100,7 +100,7 @@ enum class EOperationKind : ui32 { SchemeTx = NKikimrTxDataShard::ETransactionKind::TX_KIND_SCHEME, ReadTable = NKikimrTxDataShard::ETransactionKind::TX_KIND_SCAN, Snapshot = NKikimrTxDataShard::ETransactionKind::TX_KIND_SNAPSHOT, - DistributedErase = NKikimrTxDataShard::ETransactionKind::TX_KIND_DISTRIBUTED_ERASE, + DistributedErase = NKikimrTxDataShard::ETransactionKind::TX_KIND_DISTRIBUTED_ERASE, CommitWrites = NKikimrTxDataShard::ETransactionKind::TX_KIND_COMMIT_WRITES, // Values [100, inf) are used for internal kinds. @@ -153,7 +153,7 @@ public: bool IsSchemeTx() const { return Kind == EOperationKind::SchemeTx; } bool IsReadTable() const { return Kind == EOperationKind::ReadTable; } bool IsSnapshotTx() const { return Kind == EOperationKind::Snapshot; } - bool IsDistributedEraseTx() const { return Kind == EOperationKind::DistributedErase; } + bool IsDistributedEraseTx() const { return Kind == EOperationKind::DistributedErase; } bool IsCommitWritesTx() const { return Kind == EOperationKind::CommitWrites; } bool Exists() const { return Kind != EOperationKind::Unknown; } @@ -315,11 +315,11 @@ public: void ResetWaitingForScanFlag() { ResetFlag(TTxFlags::WaitingForScan); } bool IsWaitingForScan() const { return HasWaitingForScanFlag(); } - bool HasWaitingForAsyncJobFlag() const { return HasFlag(TTxFlags::WaitingForAsyncJob); } - void SetWaitingForAsyncJobFlag(bool val = true) { SetFlag(TTxFlags::WaitingForAsyncJob, val); } - void ResetWaitingForAsyncJobFlag() { ResetFlag(TTxFlags::WaitingForAsyncJob); } - bool IsWaitingForAsyncJob() const { return HasWaitingForAsyncJobFlag(); } - + bool HasWaitingForAsyncJobFlag() const { return HasFlag(TTxFlags::WaitingForAsyncJob); } + void SetWaitingForAsyncJobFlag(bool val = true) { SetFlag(TTxFlags::WaitingForAsyncJob, val); } + void ResetWaitingForAsyncJobFlag() { ResetFlag(TTxFlags::WaitingForAsyncJob); } + bool IsWaitingForAsyncJob() const { return HasWaitingForAsyncJobFlag(); } + bool HasWaitingForSnapshotFlag() const { return HasFlag(TTxFlags::WaitingForSnapshot); } void SetWaitingForSnapshotFlag(bool val = true) { SetFlag(TTxFlags::WaitingForSnapshot, val); } void ResetWaitingForSnapshotFlag() { ResetFlag(TTxFlags::WaitingForSnapshot); } @@ -435,7 +435,7 @@ struct TOutputOpData { using TResultPtr = THolder<TEvDataShard::TEvProposeTransactionResult>; using TDelayedAcks = TVector<THolder<IEventHandle>>; using TOutReadSets = TMap<std::pair<ui64, ui64>, TString>; // source:target -> body - using TChangeRecord = TEvChangeExchange::TEvEnqueueRecords::TRecordInfo; + using TChangeRecord = TEvChangeExchange::TEvEnqueueRecords::TRecordInfo; TResultPtr Result; // ACKs to send on successful operation completion. @@ -445,8 +445,8 @@ struct TOutputOpData { // Updates and checked locks. TLocksUpdate LocksUpdate; TLocksCache LocksAccessLog; - // Collected change records - TVector<TChangeRecord> ChangeRecords; + // Collected change records + TVector<TChangeRecord> ChangeRecords; }; struct TExecutionProfile { @@ -583,8 +583,8 @@ public: TAutoPtr<IDestructable> &AsyncJobResult() { return InputDataRef().AsyncJobResult; } void SetAsyncJobResult(TAutoPtr<IDestructable> prod) { InputDataRef().AsyncJobResult = prod; } - bool HasAsyncJobResult() const { return InputData ? (bool)InputData->AsyncJobResult : false; } - + bool HasAsyncJobResult() const { return InputData ? (bool)InputData->AsyncJobResult : false; } + //////////////////////////////////////// // OUTPUT DATA // //////////////////////////////////////// @@ -605,8 +605,8 @@ public: TLocksUpdate &LocksUpdate() { return OutputDataRef().LocksUpdate; } TLocksCache &LocksAccessLog() { return OutputDataRef().LocksAccessLog; } - TVector<TOutputOpData::TChangeRecord> &ChangeRecords() { return OutputDataRef().ChangeRecords; } - + TVector<TOutputOpData::TChangeRecord> &ChangeRecords() { return OutputDataRef().ChangeRecords; } + const NFH::TFlatHashSet<ui64> &GetAffectedLocks() const { return AffectedLocks; } void AddAffectedLock(ui64 lockTxId) { AffectedLocks.insert(lockTxId); } diff --git a/ydb/core/tx/datashard/prepare_distributed_erase_tx_in_rs_unit.cpp b/ydb/core/tx/datashard/prepare_distributed_erase_tx_in_rs_unit.cpp index 3fbbfa673e0..2a666490885 100644 --- a/ydb/core/tx/datashard/prepare_distributed_erase_tx_in_rs_unit.cpp +++ b/ydb/core/tx/datashard/prepare_distributed_erase_tx_in_rs_unit.cpp @@ -1,47 +1,47 @@ -#include "datashard_active_transaction.h" -#include "datashard_impl.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -namespace NKikimr { +#include "datashard_active_transaction.h" +#include "datashard_impl.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +namespace NKikimr { namespace NDataShard { - -class TPrepareDistributedEraseTxInRSUnit : public TExecutionUnit { -public: + +class TPrepareDistributedEraseTxInRSUnit : public TExecutionUnit { +public: TPrepareDistributedEraseTxInRSUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::PrepareDistributedEraseTxInRS, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - - EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext&) override { - Y_VERIFY(op->IsDistributedEraseTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - const auto& eraseTx = tx->GetDistributedEraseTx(); - if (!eraseTx->HasDependencies()) { - return EExecutionStatus::Executed; - } - - for (const auto& dependency : eraseTx->GetDependencies()) { - op->InReadSets().emplace(std::make_pair(dependency.GetShardId(), DataShard.TabletID()), TVector<TRSData>()); - } - - return EExecutionStatus::Executed; - } - - void Complete(TOperation::TPtr, const TActorContext&) override { - } -}; - + : TExecutionUnit(EExecutionUnitKind::PrepareDistributedEraseTxInRS, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext&) override { + Y_VERIFY(op->IsDistributedEraseTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + const auto& eraseTx = tx->GetDistributedEraseTx(); + if (!eraseTx->HasDependencies()) { + return EExecutionStatus::Executed; + } + + for (const auto& dependency : eraseTx->GetDependencies()) { + op->InReadSets().emplace(std::make_pair(dependency.GetShardId(), DataShard.TabletID()), TVector<TRSData>()); + } + + return EExecutionStatus::Executed; + } + + void Complete(TOperation::TPtr, const TActorContext&) override { + } +}; + THolder<TExecutionUnit> CreatePrepareDistributedEraseTxInRSUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TPrepareDistributedEraseTxInRSUnit(self, pipeline)); -} - + return THolder(new TPrepareDistributedEraseTxInRSUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/restore_unit.cpp b/ydb/core/tx/datashard/restore_unit.cpp index 1e70f76d7b7..6f80c11bcdf 100644 --- a/ydb/core/tx/datashard/restore_unit.cpp +++ b/ydb/core/tx/datashard/restore_unit.cpp @@ -1,96 +1,96 @@ -#include "backup_restore_common.h" -#include "execution_unit_ctors.h" -#include "import_common.h" -#include "import_s3.h" - -namespace NKikimr { +#include "backup_restore_common.h" +#include "execution_unit_ctors.h" +#include "import_common.h" +#include "import_s3.h" + +namespace NKikimr { namespace NDataShard { - -class TRestoreUnit : public TBackupRestoreUnitBase<TEvDataShard::TEvCancelRestore> { -protected: - bool IsRelevant(TActiveTransaction* tx) const override { - return tx->GetSchemeTx().HasRestore(); - } - - bool IsWaiting(TOperation::TPtr op) const override { - return op->IsWaitingForAsyncJob(); - } - - void SetWaiting(TOperation::TPtr op) override { - op->SetWaitingForAsyncJobFlag(); - } - - void ResetWaiting(TOperation::TPtr op) override { - op->ResetWaitingForAsyncJobFlag(); - } - - bool Run(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - Y_VERIFY(tx->GetSchemeTx().HasRestore()); - const auto& restore = tx->GetSchemeTx().GetRestore(); - - const ui64 tableId = restore.GetTableId(); - Y_VERIFY(DataShard.GetUserTables().contains(tableId)); - - const TTableInfo tableInfo = TTableInfo(tableId, DataShard.GetUserTables().at(tableId)); - - const auto settingsKind = restore.GetSettingsCase(); - switch (settingsKind) { + +class TRestoreUnit : public TBackupRestoreUnitBase<TEvDataShard::TEvCancelRestore> { +protected: + bool IsRelevant(TActiveTransaction* tx) const override { + return tx->GetSchemeTx().HasRestore(); + } + + bool IsWaiting(TOperation::TPtr op) const override { + return op->IsWaitingForAsyncJob(); + } + + void SetWaiting(TOperation::TPtr op) override { + op->SetWaitingForAsyncJobFlag(); + } + + void ResetWaiting(TOperation::TPtr op) override { + op->ResetWaitingForAsyncJobFlag(); + } + + bool Run(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + Y_VERIFY(tx->GetSchemeTx().HasRestore()); + const auto& restore = tx->GetSchemeTx().GetRestore(); + + const ui64 tableId = restore.GetTableId(); + Y_VERIFY(DataShard.GetUserTables().contains(tableId)); + + const TTableInfo tableInfo = TTableInfo(tableId, DataShard.GetUserTables().at(tableId)); + + const auto settingsKind = restore.GetSettingsCase(); + switch (settingsKind) { case NKikimrSchemeOp::TRestoreTask::kS3Settings: - #ifndef KIKIMR_DISABLE_S3_OPS - tx->SetAsyncJobActor(ctx.Register(CreateS3Downloader(DataShard.SelfId(), op->GetTxId(), restore, tableInfo), - TMailboxType::HTSwap, AppData(ctx)->BatchPoolId)); - break; - #else - Abort(op, ctx, "Imports from S3 are disabled"); - return false; - #endif - - default: - Abort(op, ctx, TStringBuilder() << "Unknown settings: " << static_cast<ui32>(settingsKind)); - return false; - } - - return true; - } - - bool HasResult(TOperation::TPtr op) const override { - return op->HasAsyncJobResult(); - } - - void ProcessResult(TOperation::TPtr op, const TActorContext&) override { - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - - auto* result = CheckedCast<TImportJobProduct*>(op->AsyncJobResult().Get()); - auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); - - schemeOp->Success = result->Success; - schemeOp->Error = std::move(result->Error); - schemeOp->BytesProcessed = result->BytesWritten; - schemeOp->RowsProcessed = result->RowsWritten; - - op->SetAsyncJobResult(nullptr); - tx->SetAsyncJobActor(TActorId()); - } - - void Cancel(TActiveTransaction* tx, const TActorContext& ctx) override { - tx->KillAsyncJobActor(ctx); - } - -public: + #ifndef KIKIMR_DISABLE_S3_OPS + tx->SetAsyncJobActor(ctx.Register(CreateS3Downloader(DataShard.SelfId(), op->GetTxId(), restore, tableInfo), + TMailboxType::HTSwap, AppData(ctx)->BatchPoolId)); + break; + #else + Abort(op, ctx, "Imports from S3 are disabled"); + return false; + #endif + + default: + Abort(op, ctx, TStringBuilder() << "Unknown settings: " << static_cast<ui32>(settingsKind)); + return false; + } + + return true; + } + + bool HasResult(TOperation::TPtr op) const override { + return op->HasAsyncJobResult(); + } + + void ProcessResult(TOperation::TPtr op, const TActorContext&) override { + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + + auto* result = CheckedCast<TImportJobProduct*>(op->AsyncJobResult().Get()); + auto* schemeOp = DataShard.FindSchemaTx(op->GetTxId()); + + schemeOp->Success = result->Success; + schemeOp->Error = std::move(result->Error); + schemeOp->BytesProcessed = result->BytesWritten; + schemeOp->RowsProcessed = result->RowsWritten; + + op->SetAsyncJobResult(nullptr); + tx->SetAsyncJobActor(TActorId()); + } + + void Cancel(TActiveTransaction* tx, const TActorContext& ctx) override { + tx->KillAsyncJobActor(ctx); + } + +public: TRestoreUnit(TDataShard& self, TPipeline& pipeline) - : TBase(EExecutionUnitKind::Restore, self, pipeline) - { - } - -}; // TRestoreUnit - + : TBase(EExecutionUnitKind::Restore, self, pipeline) + { + } + +}; // TRestoreUnit + THolder<TExecutionUnit> CreateRestoreUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TRestoreUnit(self, pipeline)); -} - + return THolder(new TRestoreUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/s3_common.h b/ydb/core/tx/datashard/s3_common.h index e9c98489184..a9f7c570f9c 100644 --- a/ydb/core/tx/datashard/s3_common.h +++ b/ydb/core/tx/datashard/s3_common.h @@ -1,106 +1,106 @@ -#pragma once - -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "defs.h" - -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/auth/AWSCredentials.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/StorageClass.h> - +#pragma once + +#ifndef KIKIMR_DISABLE_S3_OPS + +#include "defs.h" + +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/auth/AWSCredentials.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/StorageClass.h> + #include <ydb/core/protos/flat_scheme_op.pb.h> - -#include <util/string/builder.h> -#include <util/string/printf.h> - -namespace NKikimr { + +#include <util/string/builder.h> +#include <util/string/printf.h> + +namespace NKikimr { namespace NDataShard { - + inline Aws::Client::ClientConfiguration ConfigFromSettings(const NKikimrSchemeOp::TS3Settings& settings) { - Aws::Client::ClientConfiguration config; - - config.endpointOverride = settings.GetEndpoint(); - config.connectTimeoutMs = 10000; - config.maxConnections = 5; - - switch (settings.GetScheme()) { + Aws::Client::ClientConfiguration config; + + config.endpointOverride = settings.GetEndpoint(); + config.connectTimeoutMs = 10000; + config.maxConnections = 5; + + switch (settings.GetScheme()) { case NKikimrSchemeOp::TS3Settings::HTTP: - config.scheme = Aws::Http::Scheme::HTTP; - break; + config.scheme = Aws::Http::Scheme::HTTP; + break; case NKikimrSchemeOp::TS3Settings::HTTPS: - config.scheme = Aws::Http::Scheme::HTTPS; - break; - default: - Y_FAIL("Unknown scheme"); - } - - return config; -} - + config.scheme = Aws::Http::Scheme::HTTPS; + break; + default: + Y_FAIL("Unknown scheme"); + } + + return config; +} + inline Aws::Auth::AWSCredentials CredentialsFromSettings(const NKikimrSchemeOp::TS3Settings& settings) { - return Aws::Auth::AWSCredentials(settings.GetAccessKey(), settings.GetSecretKey()); -} - -struct TS3Settings { - Aws::Client::ClientConfiguration Config; - Aws::Auth::AWSCredentials Credentials; - TString Bucket; - TString SchemeKey; - TString DataKey; - Aws::S3::Model::StorageClass StorageClass; - -private: + return Aws::Auth::AWSCredentials(settings.GetAccessKey(), settings.GetSecretKey()); +} + +struct TS3Settings { + Aws::Client::ClientConfiguration Config; + Aws::Auth::AWSCredentials Credentials; + TString Bucket; + TString SchemeKey; + TString DataKey; + Aws::S3::Model::StorageClass StorageClass; + +private: explicit TS3Settings(const NKikimrSchemeOp::TS3Settings& settings, ui32 shard) - : Config(ConfigFromSettings(settings)) - , Credentials(CredentialsFromSettings(settings)) - , Bucket(settings.GetBucket()) - , SchemeKey(TStringBuilder() << settings.GetObjectKeyPattern() << "/scheme.pb") - , DataKey(TStringBuilder() << settings.GetObjectKeyPattern() << Sprintf("/data_%02d.csv", shard)) - , StorageClass(ConvertStorageClass(settings.GetStorageClass())) - { - Config.caPath = "/etc/ssl/certs"; - } - - static Aws::S3::Model::StorageClass ConvertStorageClass(Ydb::Export::ExportToS3Settings::StorageClass value) { - using ExportToS3Settings = Ydb::Export::ExportToS3Settings; - using AwsStorageClass = Aws::S3::Model::StorageClass; - - switch (value) { - case ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED: - return AwsStorageClass::NOT_SET; - case ExportToS3Settings::STANDARD: - return AwsStorageClass::STANDARD; - case ExportToS3Settings::REDUCED_REDUNDANCY: - return AwsStorageClass::REDUCED_REDUNDANCY; - case ExportToS3Settings::STANDARD_IA: - return AwsStorageClass::STANDARD_IA; - case ExportToS3Settings::ONEZONE_IA: - return AwsStorageClass::ONEZONE_IA; - case ExportToS3Settings::INTELLIGENT_TIERING: - return AwsStorageClass::INTELLIGENT_TIERING; - case ExportToS3Settings::GLACIER: - return AwsStorageClass::GLACIER; - case ExportToS3Settings::DEEP_ARCHIVE: - return AwsStorageClass::DEEP_ARCHIVE; - case ExportToS3Settings::OUTPOSTS: - return AwsStorageClass::OUTPOSTS; - default: - return AwsStorageClass::NOT_SET; - } - } - -public: + : Config(ConfigFromSettings(settings)) + , Credentials(CredentialsFromSettings(settings)) + , Bucket(settings.GetBucket()) + , SchemeKey(TStringBuilder() << settings.GetObjectKeyPattern() << "/scheme.pb") + , DataKey(TStringBuilder() << settings.GetObjectKeyPattern() << Sprintf("/data_%02d.csv", shard)) + , StorageClass(ConvertStorageClass(settings.GetStorageClass())) + { + Config.caPath = "/etc/ssl/certs"; + } + + static Aws::S3::Model::StorageClass ConvertStorageClass(Ydb::Export::ExportToS3Settings::StorageClass value) { + using ExportToS3Settings = Ydb::Export::ExportToS3Settings; + using AwsStorageClass = Aws::S3::Model::StorageClass; + + switch (value) { + case ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED: + return AwsStorageClass::NOT_SET; + case ExportToS3Settings::STANDARD: + return AwsStorageClass::STANDARD; + case ExportToS3Settings::REDUCED_REDUNDANCY: + return AwsStorageClass::REDUCED_REDUNDANCY; + case ExportToS3Settings::STANDARD_IA: + return AwsStorageClass::STANDARD_IA; + case ExportToS3Settings::ONEZONE_IA: + return AwsStorageClass::ONEZONE_IA; + case ExportToS3Settings::INTELLIGENT_TIERING: + return AwsStorageClass::INTELLIGENT_TIERING; + case ExportToS3Settings::GLACIER: + return AwsStorageClass::GLACIER; + case ExportToS3Settings::DEEP_ARCHIVE: + return AwsStorageClass::DEEP_ARCHIVE; + case ExportToS3Settings::OUTPOSTS: + return AwsStorageClass::OUTPOSTS; + default: + return AwsStorageClass::NOT_SET; + } + } + +public: static TS3Settings FromBackupTask(const NKikimrSchemeOp::TBackupTask& task) { - return TS3Settings(task.GetS3Settings(), task.GetShardNum()); - } - + return TS3Settings(task.GetS3Settings(), task.GetShardNum()); + } + static TS3Settings FromRestoreTask(const NKikimrSchemeOp::TRestoreTask& task) { - return TS3Settings(task.GetS3Settings(), task.GetShardNum()); - } - -}; // TS3Settings - + return TS3Settings(task.GetS3Settings(), task.GetShardNum()); + } + +}; // TS3Settings + } // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS +} // NKikimr + +#endif // KIKIMR_DISABLE_S3_OPS diff --git a/ydb/core/tx/datashard/snapshot_key.h b/ydb/core/tx/datashard/snapshot_key.h index fb59be9f449..7a742a6e8aa 100644 --- a/ydb/core/tx/datashard/snapshot_key.h +++ b/ydb/core/tx/datashard/snapshot_key.h @@ -1,101 +1,101 @@ -#pragma once - -#include <util/generic/fwd.h> - -#include <tuple> - -namespace NKikimr { -namespace NDataShard { - -namespace NPrivate { - -template <typename T, size_t I = std::tuple_size<T>::value - 1> -size_t HashTuple(const T& tp) { - if constexpr (I != 0) { - return std::get<I>(tp) + 31 * HashTuple<T, I - 1>(tp); - } else { - return std::get<I>(tp); - } -} - -template <typename TDerived> -struct TCommonOps { - friend inline bool operator<(const TDerived& a, const TDerived& b) { - return a.ToTuple() < b.ToTuple(); - } - - friend inline bool operator==(const TDerived& a, const TDerived& b) { - return a.ToTuple() == b.ToTuple(); - } - - explicit operator size_t() const noexcept { - return HashTuple(static_cast<const TDerived*>(this)->ToTuple()); - } -}; - -} // NPrivate - -struct TSnapshotTableKey : public NPrivate::TCommonOps<TSnapshotTableKey> { - ui64 OwnerId = 0; - ui64 PathId = 0; - - TSnapshotTableKey() = default; - - TSnapshotTableKey(ui64 ownerId, ui64 pathId) - : OwnerId(ownerId) - , PathId(pathId) - { } - - auto ToTuple() const { - return std::make_tuple(OwnerId, PathId); - } - - using TCommonOps<TSnapshotTableKey>::operator size_t; -}; - -struct TDataSnapshotKey - : public TSnapshotTableKey - , public NPrivate::TCommonOps<TDataSnapshotKey> -{ - ui64 Step = 0; - ui64 TxId = 0; - - TDataSnapshotKey() = default; - - TDataSnapshotKey(ui64 ownerId, ui64 pathId, ui64 step, ui64 txId) - : TSnapshotTableKey(ownerId, pathId) - , Step(step) - , TxId(txId) - { } - - auto ToTuple() const { - return std::tuple_cat(TSnapshotTableKey::ToTuple(), std::make_tuple(Step, TxId)); - } - - using TCommonOps<TDataSnapshotKey>::operator size_t; -}; - -using TSnapshotKey = TDataSnapshotKey; - -struct TSchemaSnapshotKey - : public TSnapshotTableKey - , public NPrivate::TCommonOps<TSchemaSnapshotKey> -{ - ui64 Version = 0; - - TSchemaSnapshotKey() = default; - - TSchemaSnapshotKey(ui64 ownerId, ui64 pathId, ui64 version) - : TSnapshotTableKey(ownerId, pathId) - , Version(version) - { } - - auto ToTuple() const { - return std::tuple_cat(TSnapshotTableKey::ToTuple(), std::make_tuple(Version)); - } - - using TCommonOps<TSchemaSnapshotKey>::operator size_t; -}; - -} // NDataShard -} // NKikimr +#pragma once + +#include <util/generic/fwd.h> + +#include <tuple> + +namespace NKikimr { +namespace NDataShard { + +namespace NPrivate { + +template <typename T, size_t I = std::tuple_size<T>::value - 1> +size_t HashTuple(const T& tp) { + if constexpr (I != 0) { + return std::get<I>(tp) + 31 * HashTuple<T, I - 1>(tp); + } else { + return std::get<I>(tp); + } +} + +template <typename TDerived> +struct TCommonOps { + friend inline bool operator<(const TDerived& a, const TDerived& b) { + return a.ToTuple() < b.ToTuple(); + } + + friend inline bool operator==(const TDerived& a, const TDerived& b) { + return a.ToTuple() == b.ToTuple(); + } + + explicit operator size_t() const noexcept { + return HashTuple(static_cast<const TDerived*>(this)->ToTuple()); + } +}; + +} // NPrivate + +struct TSnapshotTableKey : public NPrivate::TCommonOps<TSnapshotTableKey> { + ui64 OwnerId = 0; + ui64 PathId = 0; + + TSnapshotTableKey() = default; + + TSnapshotTableKey(ui64 ownerId, ui64 pathId) + : OwnerId(ownerId) + , PathId(pathId) + { } + + auto ToTuple() const { + return std::make_tuple(OwnerId, PathId); + } + + using TCommonOps<TSnapshotTableKey>::operator size_t; +}; + +struct TDataSnapshotKey + : public TSnapshotTableKey + , public NPrivate::TCommonOps<TDataSnapshotKey> +{ + ui64 Step = 0; + ui64 TxId = 0; + + TDataSnapshotKey() = default; + + TDataSnapshotKey(ui64 ownerId, ui64 pathId, ui64 step, ui64 txId) + : TSnapshotTableKey(ownerId, pathId) + , Step(step) + , TxId(txId) + { } + + auto ToTuple() const { + return std::tuple_cat(TSnapshotTableKey::ToTuple(), std::make_tuple(Step, TxId)); + } + + using TCommonOps<TDataSnapshotKey>::operator size_t; +}; + +using TSnapshotKey = TDataSnapshotKey; + +struct TSchemaSnapshotKey + : public TSnapshotTableKey + , public NPrivate::TCommonOps<TSchemaSnapshotKey> +{ + ui64 Version = 0; + + TSchemaSnapshotKey() = default; + + TSchemaSnapshotKey(ui64 ownerId, ui64 pathId, ui64 version) + : TSnapshotTableKey(ownerId, pathId) + , Version(version) + { } + + auto ToTuple() const { + return std::tuple_cat(TSnapshotTableKey::ToTuple(), std::make_tuple(Version)); + } + + using TCommonOps<TSchemaSnapshotKey>::operator size_t; +}; + +} // NDataShard +} // NKikimr diff --git a/ydb/core/tx/datashard/store_distributed_erase_tx_unit.cpp b/ydb/core/tx/datashard/store_distributed_erase_tx_unit.cpp index 152d343de30..bc78972d0d8 100644 --- a/ydb/core/tx/datashard/store_distributed_erase_tx_unit.cpp +++ b/ydb/core/tx/datashard/store_distributed_erase_tx_unit.cpp @@ -1,42 +1,42 @@ -#include "datashard_active_transaction.h" -#include "datashard_pipeline.h" -#include "execution_unit_ctors.h" - -namespace NKikimr { +#include "datashard_active_transaction.h" +#include "datashard_pipeline.h" +#include "execution_unit_ctors.h" + +namespace NKikimr { namespace NDataShard { - -class TStoreDistributedEraseTxUnit : public TExecutionUnit { -public: + +class TStoreDistributedEraseTxUnit : public TExecutionUnit { +public: TStoreDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) - : TExecutionUnit(EExecutionUnitKind::StoreDistributedEraseTx, false, self, pipeline) - { - } - - bool IsReadyToExecute(TOperation::TPtr) const override { - return true; - } - + : TExecutionUnit(EExecutionUnitKind::StoreDistributedEraseTx, false, self, pipeline) + { + } + + bool IsReadyToExecute(TOperation::TPtr) const override { + return true; + } + EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override { - Y_VERIFY(op->IsDistributedEraseTx()); - - TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); - Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); - + Y_VERIFY(op->IsDistributedEraseTx()); + + TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); + Y_VERIFY_S(tx, "cannot cast operation of kind " << op->GetKind()); + Pipeline.ProposeTx(op, tx->GetTxBody(), txc, ctx); - tx->ClearTxBody(); - tx->ClearDistributedEraseTx(); - + tx->ClearTxBody(); + tx->ClearDistributedEraseTx(); + return EExecutionStatus::DelayCompleteNoMoreRestarts; - } - + } + void Complete(TOperation::TPtr op, const TActorContext& ctx) override { Pipeline.ProposeComplete(op, ctx); - } -}; - + } +}; + THolder<TExecutionUnit> CreateStoreDistributedEraseTxUnit(TDataShard& self, TPipeline& pipeline) { - return THolder(new TStoreDistributedEraseTxUnit(self, pipeline)); -} - + return THolder(new TStoreDistributedEraseTxUnit(self, pipeline)); +} + } // namespace NDataShard -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/tx/datashard/store_scheme_tx_unit.cpp b/ydb/core/tx/datashard/store_scheme_tx_unit.cpp index fa54f12b845..6b1d8af9ed9 100644 --- a/ydb/core/tx/datashard/store_scheme_tx_unit.cpp +++ b/ydb/core/tx/datashard/store_scheme_tx_unit.cpp @@ -76,9 +76,9 @@ EExecutionStatus TStoreSchemeTxUnit::Execute(TOperation::TPtr op, DataShard.PersistProcessingParams(tx->GetProcessingParams(), txc); } - TSchemaOperation schemeOp(op->GetTxId(), tx->GetSchemeTxType(), op->GetTarget(), + TSchemaOperation schemeOp(op->GetTxId(), tx->GetSchemeTxType(), op->GetTarget(), tx->GetSchemeShardId(), op->GetMinStep(), op->GetMaxStep(), - 0, op->IsReadOnly(), false, TString(), 0, 0); + 0, op->IsReadOnly(), false, TString(), 0, 0); Pipeline.ProposeSchemeTx(schemeOp, txc); Pipeline.ProposeTx(op, tx->GetTxBody(), txc, ctx); diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 5d7ee4a186b..9c98e8e79b0 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -6,34 +6,34 @@ OWNER( ) SRCS( - alter_cdc_stream_unit.cpp + alter_cdc_stream_unit.cpp alter_table_unit.cpp backup_unit.cpp build_and_wait_dependencies_unit.cpp build_data_tx_out_rs_unit.cpp - build_distributed_erase_tx_out_rs_unit.cpp + build_distributed_erase_tx_out_rs_unit.cpp build_kqp_data_tx_out_rs_unit.cpp build_scheme_tx_out_rs_unit.cpp - change_collector_async_index.cpp - change_collector_base.cpp - change_collector_cdc_stream.cpp - change_collector_helpers.cpp - change_collector.cpp - change_exchange.cpp - change_exchange_split.cpp - change_record.cpp - change_sender.cpp - change_sender_async_index.cpp - change_sender_cdc_stream.cpp - change_sender_common_ops.cpp + change_collector_async_index.cpp + change_collector_base.cpp + change_collector_cdc_stream.cpp + change_collector_helpers.cpp + change_collector.cpp + change_exchange.cpp + change_exchange_split.cpp + change_record.cpp + change_sender.cpp + change_sender_async_index.cpp + change_sender_cdc_stream.cpp + change_sender_common_ops.cpp check_commit_writes_tx_unit.cpp check_data_tx_unit.cpp - check_distributed_erase_tx_unit.cpp + check_distributed_erase_tx_unit.cpp check_scheme_tx_unit.cpp check_snapshot_tx_unit.cpp complete_data_tx_unit.cpp completed_operations_unit.cpp - create_cdc_stream_unit.cpp + create_cdc_stream_unit.cpp create_persistent_snapshot_unit.cpp create_table_unit.cpp create_volatile_snapshot_unit.cpp @@ -44,16 +44,16 @@ SRCS( datashard__cleanup_borrowed.cpp datashard__cleanup_in_rs.cpp datashard__cleanup_tx.cpp - datashard__conditional_erase_rows.cpp + datashard__conditional_erase_rows.cpp datashard__engine_host.cpp datashard__engine_host.h datashard__get_state_tx.cpp datashard__schema_changed.cpp datashard__migrate_schemeshard.cpp datashard__init.cpp - datashard__monitoring.cpp + datashard__monitoring.cpp datashard__mon_reset_schema_version.cpp - datashard__op_rows.cpp + datashard__op_rows.cpp datashard__plan_step.cpp datashard__progress_resend_rs.cpp datashard__progress_tx.cpp @@ -62,16 +62,16 @@ SRCS( datashard__read_iterator.cpp datashard__read_columns.cpp datashard__s3.cpp - datashard__s3_download_txs.cpp - datashard__s3_upload_txs.cpp + datashard__s3_download_txs.cpp + datashard__s3_upload_txs.cpp datashard__kqp_scan.cpp datashard__snapshot_txs.cpp datashard__stats.cpp datashard__store_table_path.cpp datashard__store_scan_state.cpp - datashard_change_receiving.cpp - datashard_change_sender_activation.cpp - datashard_change_sending.cpp + datashard_change_receiving.cpp + datashard_change_sender_activation.cpp + datashard_change_sending.cpp datashard_counters.cpp datashard_loans.cpp datashard_locks.h @@ -85,23 +85,23 @@ SRCS( datashard_outreadset.h datashard_active_transaction.cpp datashard_active_transaction.h - datashard_common_upload.cpp + datashard_common_upload.cpp datashard_direct_transaction.cpp datashard_direct_transaction.h - datashard_direct_erase.cpp - datashard_direct_upload.cpp - datashard_distributed_erase.cpp + datashard_direct_erase.cpp + datashard_direct_upload.cpp + datashard_distributed_erase.cpp datashard_failpoints.cpp datashard_failpoints.h datashard_dep_tracker.cpp datashard_dep_tracker.h datashard_pipeline.cpp datashard_pipeline.h - datashard_s3_downloads.cpp - datashard_s3_uploads.cpp - datashard_schema_snapshots.cpp + datashard_s3_downloads.cpp + datashard_s3_uploads.cpp + datashard_schema_snapshots.cpp datashard_snapshots.cpp - datashard_unsafe_upload.cpp + datashard_unsafe_upload.cpp datashard_user_table.cpp datashard_user_table.h datashard_impl.h @@ -123,29 +123,29 @@ SRCS( datashard.cpp datashard.h defs.h - direct_tx_unit.cpp - drop_cdc_stream_unit.cpp + direct_tx_unit.cpp + drop_cdc_stream_unit.cpp drop_index_notice_unit.cpp drop_persistent_snapshot_unit.cpp drop_table_unit.cpp drop_volatile_snapshot_unit.cpp - erase_rows_condition.cpp + erase_rows_condition.cpp execute_commit_writes_tx_unit.cpp execute_data_tx_unit.cpp - execute_distributed_erase_tx_unit.cpp + execute_distributed_erase_tx_unit.cpp execute_kqp_data_tx_unit.cpp execute_kqp_scan_tx_unit.cpp execution_unit.cpp execution_unit.h execution_unit_ctors.h execution_unit_kind.h - export_common.cpp + export_common.cpp export_iface.cpp export_iface.h - export_scan.cpp - finalize_build_index_unit.cpp + export_scan.cpp + finalize_build_index_unit.cpp finish_propose_unit.cpp - initiate_build_index_unit.cpp + initiate_build_index_unit.cpp key_conflicts.cpp key_conflicts.h load_and_wait_in_rs_unit.cpp @@ -157,7 +157,7 @@ SRCS( operation.h plan_queue_unit.cpp prepare_data_tx_in_rs_unit.cpp - prepare_distributed_erase_tx_in_rs_unit.cpp + prepare_distributed_erase_tx_in_rs_unit.cpp prepare_kqp_data_tx_in_rs_unit.cpp prepare_scheme_tx_in_rs_unit.cpp progress_queue.h @@ -170,12 +170,12 @@ SRCS( range_ops.cpp range_treap.cpp read_iterator.h - restore_unit.cpp + restore_unit.cpp setup_sys_locks.h store_and_send_out_rs_unit.cpp store_commit_writes_tx_unit.cpp store_data_tx_unit.cpp - store_distributed_erase_tx_unit.cpp + store_distributed_erase_tx_unit.cpp store_scheme_tx_unit.cpp store_snapshot_tx_unit.cpp wait_for_plan_unit.cpp @@ -183,15 +183,15 @@ SRCS( upload_stats.cpp ) -GENERATE_ENUM_SERIALIZATION(change_exchange.h) +GENERATE_ENUM_SERIALIZATION(change_exchange.h) -GENERATE_ENUM_SERIALIZATION(change_record.h) +GENERATE_ENUM_SERIALIZATION(change_record.h) GENERATE_ENUM_SERIALIZATION(datashard.h) GENERATE_ENUM_SERIALIZATION(datashard_active_transaction.h) -GENERATE_ENUM_SERIALIZATION(datashard_s3_upload.h) +GENERATE_ENUM_SERIALIZATION(datashard_s3_upload.h) GENERATE_ENUM_SERIALIZATION(execution_unit.h) @@ -237,14 +237,14 @@ IF (OS_WINDOWS) CFLAGS( -DKIKIMR_DISABLE_S3_OPS ) -ELSE() +ELSE() SRCS( - export_s3_buffer.cpp - export_s3_uploader.cpp + export_s3_buffer.cpp + export_s3_uploader.cpp import_s3.cpp ) -ENDIF() - +ENDIF() + END() RECURSE_FOR_TESTS( diff --git a/ydb/core/tx/long_tx_service/long_tx_service_ut.cpp b/ydb/core/tx/long_tx_service/long_tx_service_ut.cpp index 284d0ba79a0..1a2375e15f3 100644 --- a/ydb/core/tx/long_tx_service/long_tx_service_ut.cpp +++ b/ydb/core/tx/long_tx_service/long_tx_service_ut.cpp @@ -61,8 +61,8 @@ Y_UNIT_TEST_SUITE(LongTxService) { void StartSchemeCache(TTestActorRuntime& runtime, const TString& root = DOMAIN1_NAME) { for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) { - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); - cacheConfig->Roots.emplace_back(1, SCHEME_SHARD1_ID, root); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); + cacheConfig->Roots.emplace_back(1, SCHEME_SHARD1_ID, root); cacheConfig->Counters = new NMonitoring::TDynamicCounters(); IActor* schemeCache = CreateSchemeBoardSchemeCache(cacheConfig.Get()); diff --git a/ydb/core/tx/replication/controller/controller.cpp b/ydb/core/tx/replication/controller/controller.cpp index 77256b32636..4dc42589310 100644 --- a/ydb/core/tx/replication/controller/controller.cpp +++ b/ydb/core/tx/replication/controller/controller.cpp @@ -1,143 +1,143 @@ -#include "controller.h" -#include "controller_impl.h" - +#include "controller.h" +#include "controller_impl.h" + #include <ydb/core/engine/minikql/flat_local_tx_factory.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -TController::TController(const TActorId& tablet, TTabletStorageInfo* info) - : TActor(&TThis::StateInit) - , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) - , LogPrefix(this) -{ -} - -void TController::OnDetach(const TActorContext& ctx) { - CLOG_T(ctx, "OnDetach"); - Die(ctx); -} - -void TController::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) { - CLOG_T(ctx, "OnTabletDead"); - Die(ctx); -} - -void TController::OnActivateExecutor(const TActorContext& ctx) { - CLOG_T(ctx, "OnActivateExecutor"); - RunTxInitSchema(ctx); -} - -void TController::DefaultSignalTabletActive(const TActorContext&) { - // nop -} - -STFUNC(TController::StateInit) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvPoison, Handle); - default: - return StateInitImpl(ev, ctx); - } -} - -STFUNC(TController::StateZombie) { - StateInitImpl(ev, ctx); -} - -STFUNC(TController::StateWork) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvController::TEvCreateReplication, Handle); - HFunc(TEvController::TEvDropReplication, Handle); - HFunc(TEvPrivate::TEvDiscoveryResult, Handle); - HFunc(TEvPrivate::TEvAssignStreamName, Handle); - HFunc(TEvPrivate::TEvCreateStreamResult, Handle); - HFunc(TEvPrivate::TEvCreateDstResult, Handle); - HFunc(TEvents::TEvPoison, Handle); - } -} - -void TController::SwitchToWork(const TActorContext& ctx) { - CLOG_T(ctx, "SwitchToWork"); - - SignalTabletActive(ctx); - Become(&TThis::StateWork); - - for (auto& [_, replication] : Replications) { - replication->Progress(ctx); - } -} - -void TController::Reset() { - SysParams.Reset(); - Replications.clear(); - ReplicationsByPathId.clear(); -} - -void TController::Handle(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxCreateReplication(ev, ctx); -} - -void TController::Handle(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxDropReplication(ev, ctx); -} - -void TController::Handle(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxDiscoveryResult(ev, ctx); -} - -void TController::Handle(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxAssignStreamName(ev, ctx); -} - -void TController::Handle(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxCreateStreamResult(ev, ctx); -} - -void TController::Handle(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - RunTxCreateDstResult(ev, ctx); -} - -void TController::Handle(TEvents::TEvPoison::TPtr& ev, const TActorContext& ctx) { - CLOG_T(ctx, "Handle " << ev->Get()->ToString()); - - for (auto& [_, replication] : Replications) { - replication->Shutdown(ctx); - } - - Send(Tablet(), new TEvents::TEvPoison()); - Become(&TThis::StateZombie); -} - -TReplication::TPtr TController::Find(ui64 id) { - auto it = Replications.find(id); - if (it == Replications.end()) { - return nullptr; - } - - return it->second; -} - -TReplication::TPtr TController::Find(const TPathId& pathId) { - auto it = ReplicationsByPathId.find(pathId); - if (it == ReplicationsByPathId.end()) { - return nullptr; - } - - return it->second; -} - -} // NController - -IActor* CreateController(const TActorId& tablet, TTabletStorageInfo* info) { - return new NController::TController(tablet, info); -} - -} // NReplication -} // NKikimr + +namespace NKikimr { +namespace NReplication { +namespace NController { + +TController::TController(const TActorId& tablet, TTabletStorageInfo* info) + : TActor(&TThis::StateInit) + , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) + , LogPrefix(this) +{ +} + +void TController::OnDetach(const TActorContext& ctx) { + CLOG_T(ctx, "OnDetach"); + Die(ctx); +} + +void TController::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) { + CLOG_T(ctx, "OnTabletDead"); + Die(ctx); +} + +void TController::OnActivateExecutor(const TActorContext& ctx) { + CLOG_T(ctx, "OnActivateExecutor"); + RunTxInitSchema(ctx); +} + +void TController::DefaultSignalTabletActive(const TActorContext&) { + // nop +} + +STFUNC(TController::StateInit) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoison, Handle); + default: + return StateInitImpl(ev, ctx); + } +} + +STFUNC(TController::StateZombie) { + StateInitImpl(ev, ctx); +} + +STFUNC(TController::StateWork) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvController::TEvCreateReplication, Handle); + HFunc(TEvController::TEvDropReplication, Handle); + HFunc(TEvPrivate::TEvDiscoveryResult, Handle); + HFunc(TEvPrivate::TEvAssignStreamName, Handle); + HFunc(TEvPrivate::TEvCreateStreamResult, Handle); + HFunc(TEvPrivate::TEvCreateDstResult, Handle); + HFunc(TEvents::TEvPoison, Handle); + } +} + +void TController::SwitchToWork(const TActorContext& ctx) { + CLOG_T(ctx, "SwitchToWork"); + + SignalTabletActive(ctx); + Become(&TThis::StateWork); + + for (auto& [_, replication] : Replications) { + replication->Progress(ctx); + } +} + +void TController::Reset() { + SysParams.Reset(); + Replications.clear(); + ReplicationsByPathId.clear(); +} + +void TController::Handle(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxCreateReplication(ev, ctx); +} + +void TController::Handle(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxDropReplication(ev, ctx); +} + +void TController::Handle(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxDiscoveryResult(ev, ctx); +} + +void TController::Handle(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxAssignStreamName(ev, ctx); +} + +void TController::Handle(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxCreateStreamResult(ev, ctx); +} + +void TController::Handle(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + RunTxCreateDstResult(ev, ctx); +} + +void TController::Handle(TEvents::TEvPoison::TPtr& ev, const TActorContext& ctx) { + CLOG_T(ctx, "Handle " << ev->Get()->ToString()); + + for (auto& [_, replication] : Replications) { + replication->Shutdown(ctx); + } + + Send(Tablet(), new TEvents::TEvPoison()); + Become(&TThis::StateZombie); +} + +TReplication::TPtr TController::Find(ui64 id) { + auto it = Replications.find(id); + if (it == Replications.end()) { + return nullptr; + } + + return it->second; +} + +TReplication::TPtr TController::Find(const TPathId& pathId) { + auto it = ReplicationsByPathId.find(pathId); + if (it == ReplicationsByPathId.end()) { + return nullptr; + } + + return it->second; +} + +} // NController + +IActor* CreateController(const TActorId& tablet, TTabletStorageInfo* info) { + return new NController::TController(tablet, info); +} + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/controller.h b/ydb/core/tx/replication/controller/controller.h index fe15840878d..8a7594472a1 100644 --- a/ydb/core/tx/replication/controller/controller.h +++ b/ydb/core/tx/replication/controller/controller.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/core/base/blobstorage.h> #include <ydb/core/base/defs.h> - -namespace NKikimr { -namespace NReplication { - -IActor* CreateController(const TActorId& tablet, TTabletStorageInfo* info); - -} // NReplication -} // NKikimr + +namespace NKikimr { +namespace NReplication { + +IActor* CreateController(const TActorId& tablet, TTabletStorageInfo* info); + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/controller_impl.h b/ydb/core/tx/replication/controller/controller_impl.h index 9f429906386..145fb5b0f3a 100644 --- a/ydb/core/tx/replication/controller/controller_impl.h +++ b/ydb/core/tx/replication/controller/controller_impl.h @@ -1,123 +1,123 @@ -#pragma once - -#include "logging.h" -#include "private_events.h" -#include "public_events.h" -#include "replication.h" -#include "schema.h" -#include "sys_params.h" - +#pragma once + +#include "logging.h" +#include "private_events.h" +#include "public_events.h" +#include "replication.h" +#include "schema.h" +#include "sys_params.h" + #include <ydb/core/base/blobstorage.h> #include <ydb/core/base/defs.h> #include <ydb/core/protos/counters_replication.pb.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> -#include <ydb/core/util/yverify_stream.h> - -#include <util/generic/hash.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController - : public TActor<TController> - , public NTabletFlatExecutor::TTabletExecutedFlat -{ -public: - class TTxBase: public NTabletFlatExecutor::TTransactionBase<TController> { - public: - TTxBase(const TString& name, TController* self) - : TTransactionBase(self) - , LogPrefix(self, name) - { - } - - protected: - const TTabletLogPrefix LogPrefix; - }; - -private: - using Schema = TControllerSchema; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::REPLICATION_CONTROLLER_ACTOR; - } - - explicit TController(const TActorId& tablet, TTabletStorageInfo* info); - -private: - // tablet overrides - void OnDetach(const TActorContext& ctx) override; - void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override; - void OnActivateExecutor(const TActorContext& ctx) override; - void DefaultSignalTabletActive(const TActorContext& ctx) override; - - // state functions - STFUNC(StateInit); - STFUNC(StateZombie); - STFUNC(StateWork); - - void SwitchToWork(const TActorContext& ctx); - void Reset(); - - // handlers - void Handle(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx); - void Handle(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvents::TEvPoison::TPtr& ev, const TActorContext& ctx); - - // local transactions - class TTxInitSchema; - class TTxInit; - class TTxCreateReplication; - class TTxDropReplication; - class TTxDiscoveryResult; - class TTxAssignStreamName; - class TTxCreateStreamResult; - class TTxCreateDstResult; - - // tx runners - void RunTxInitSchema(const TActorContext& ctx); - void RunTxInit(const TActorContext& ctx); - void RunTxCreateReplication(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx); - void RunTxDropReplication(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx); - void RunTxDiscoveryResult(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx); - void RunTxAssignStreamName(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx); - void RunTxCreateStreamResult(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx); - void RunTxCreateDstResult(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx); - - // other - template <typename T> - TReplication::TPtr Add(ui64 id, const TPathId& pathId, T&& config) { - auto replication = MakeIntrusive<TReplication>(id, pathId, std::forward<T>(config)); - { - const auto res = Replications.emplace(id, replication); - Y_VERIFY_S(res.second, "Duplication replication: " << id); - } - { - const auto res = ReplicationsByPathId.emplace(pathId, replication); - Y_VERIFY_S(res.second, "Duplication replication: " << pathId); - } - - return replication; - } - - TReplication::TPtr Find(ui64 id); - TReplication::TPtr Find(const TPathId& pathId); - -private: - const TTabletLogPrefix LogPrefix; - - TSysParams SysParams; - THashMap<ui64, TReplication::TPtr> Replications; - THashMap<TPathId, TReplication::TPtr> ReplicationsByPathId; - -}; // TController - -} // NController -} // NReplication -} // NKikimr +#include <ydb/core/util/yverify_stream.h> + +#include <util/generic/hash.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController + : public TActor<TController> + , public NTabletFlatExecutor::TTabletExecutedFlat +{ +public: + class TTxBase: public NTabletFlatExecutor::TTransactionBase<TController> { + public: + TTxBase(const TString& name, TController* self) + : TTransactionBase(self) + , LogPrefix(self, name) + { + } + + protected: + const TTabletLogPrefix LogPrefix; + }; + +private: + using Schema = TControllerSchema; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::REPLICATION_CONTROLLER_ACTOR; + } + + explicit TController(const TActorId& tablet, TTabletStorageInfo* info); + +private: + // tablet overrides + void OnDetach(const TActorContext& ctx) override; + void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override; + void OnActivateExecutor(const TActorContext& ctx) override; + void DefaultSignalTabletActive(const TActorContext& ctx) override; + + // state functions + STFUNC(StateInit); + STFUNC(StateZombie); + STFUNC(StateWork); + + void SwitchToWork(const TActorContext& ctx); + void Reset(); + + // handlers + void Handle(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx); + void Handle(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvents::TEvPoison::TPtr& ev, const TActorContext& ctx); + + // local transactions + class TTxInitSchema; + class TTxInit; + class TTxCreateReplication; + class TTxDropReplication; + class TTxDiscoveryResult; + class TTxAssignStreamName; + class TTxCreateStreamResult; + class TTxCreateDstResult; + + // tx runners + void RunTxInitSchema(const TActorContext& ctx); + void RunTxInit(const TActorContext& ctx); + void RunTxCreateReplication(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx); + void RunTxDropReplication(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx); + void RunTxDiscoveryResult(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx); + void RunTxAssignStreamName(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx); + void RunTxCreateStreamResult(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx); + void RunTxCreateDstResult(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx); + + // other + template <typename T> + TReplication::TPtr Add(ui64 id, const TPathId& pathId, T&& config) { + auto replication = MakeIntrusive<TReplication>(id, pathId, std::forward<T>(config)); + { + const auto res = Replications.emplace(id, replication); + Y_VERIFY_S(res.second, "Duplication replication: " << id); + } + { + const auto res = ReplicationsByPathId.emplace(pathId, replication); + Y_VERIFY_S(res.second, "Duplication replication: " << pathId); + } + + return replication; + } + + TReplication::TPtr Find(ui64 id); + TReplication::TPtr Find(const TPathId& pathId); + +private: + const TTabletLogPrefix LogPrefix; + + TSysParams SysParams; + THashMap<ui64, TReplication::TPtr> Replications; + THashMap<TPathId, TReplication::TPtr> ReplicationsByPathId; + +}; // TController + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/discoverer.cpp b/ydb/core/tx/replication/controller/discoverer.cpp index 6ec1c865ce7..6d4e1dec0b1 100644 --- a/ydb/core/tx/replication/controller/discoverer.cpp +++ b/ydb/core/tx/replication/controller/discoverer.cpp @@ -1,152 +1,152 @@ -#include "discoverer.h" -#include "logging.h" -#include "private_events.h" -#include "util.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> - -#include <util/generic/hash_set.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TDiscoverer: public TActorBootstrapped<TDiscoverer> { - void DescribePath(ui32 idx) { - Y_VERIFY(idx < Paths.size()); - Send(YdbProxy, new TEvYdbProxy::TEvDescribePathRequest(Paths.at(idx).first, {}), 0, idx); - Pending.insert(idx); - } - - void Handle(TEvYdbProxy::TEvDescribePathResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - - auto it = Pending.find(ev->Cookie); - if (it == Pending.end()) { - LOG_W("Unknown describe response" - << ": cookie# " << ev->Cookie); - return; - } - - Y_VERIFY(*it < Paths.size()); - const auto& path = Paths.at(*it); - - const auto& result = ev->Get()->Result; - if (result.IsSuccess()) { - LOG_D("Describe succeeded" - << ": path# " << path.first); - - auto entry = result.GetEntry(); - entry.Name = path.first; // replace by full path - - if (const auto kind = TryTargetKindFromEntryType(entry.Type)) { - LOG_I("Add target" - << ": path# " << path.first - << ", kind# " << kind); - ToAdd.emplace_back(std::move(entry), path.second); - } else { - LOG_W("Unsupported entry type" - << ": path# " << path.first - << ", type# " << entry.Type); - - NYql::TIssues issues; - issues.AddIssue(TStringBuilder() << "Unsupported entry type: " << entry.Type); - Failed.emplace_back(path.first, NYdb::TStatus(NYdb::EStatus::UNSUPPORTED, std::move(issues))); - } - } else { - LOG_E("Describe failed" - << ": path# " << path.first); - - if (IsRetryableError(result)) { - return Retry(*it); - } else { - Failed.emplace_back(path.first, result); - } - } - - Pending.erase(it); - if (Pending) { - return; - } - - if (Failed) { - Send(Parent, new TEvPrivate::TEvDiscoveryResult(ReplicationId, std::move(Failed))); - } else { - Send(Parent, new TEvPrivate::TEvDiscoveryResult(ReplicationId, std::move(ToAdd), {})); - } - - PassAway(); - } - - void Retry(ui32 idx) { - if (ToRetry.empty()) { - Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup); - } - - ToRetry.insert(idx); - } - - void Retry() { - for (const ui32 idx : ToRetry) { - DescribePath(idx); - } - - ToRetry.clear(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::REPLICATION_CONTROLLER_DISCOVERER; - } - - explicit TDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, TVector<std::pair<TString, TString>>&& paths) - : Parent(parent) - , ReplicationId(rid) - , YdbProxy(proxy) - , Paths(std::move(paths)) - , LogPrefix("Discoverer", ReplicationId) - { - } - - void Bootstrap() { - for (ui32 i = 0; i < Paths.size(); ++i) { - DescribePath(i); - } - - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvYdbProxy::TEvDescribePathResponse, Handle); - sFunc(TEvents::TEvWakeup, Retry); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const ui64 ReplicationId; - const TActorId YdbProxy; - const TVector<std::pair<TString, TString>> Paths; - const TActorLogPrefix LogPrefix; - - THashSet<ui32> Pending; - THashSet<ui32> ToRetry; - TVector<TEvPrivate::TEvDiscoveryResult::TAddEntry> ToAdd; - TVector<TEvPrivate::TEvDiscoveryResult::TFailedEntry> Failed; - -}; // TDiscoverer - -IActor* CreateDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, - TVector<std::pair<TString, TString>>&& specificPaths) -{ - return new TDiscoverer(parent, rid, proxy, std::move(specificPaths)); -} - -} // NController -} // NReplication -} // NKikimr +#include "discoverer.h" +#include "logging.h" +#include "private_events.h" +#include "util.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> + +#include <util/generic/hash_set.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TDiscoverer: public TActorBootstrapped<TDiscoverer> { + void DescribePath(ui32 idx) { + Y_VERIFY(idx < Paths.size()); + Send(YdbProxy, new TEvYdbProxy::TEvDescribePathRequest(Paths.at(idx).first, {}), 0, idx); + Pending.insert(idx); + } + + void Handle(TEvYdbProxy::TEvDescribePathResponse::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + + auto it = Pending.find(ev->Cookie); + if (it == Pending.end()) { + LOG_W("Unknown describe response" + << ": cookie# " << ev->Cookie); + return; + } + + Y_VERIFY(*it < Paths.size()); + const auto& path = Paths.at(*it); + + const auto& result = ev->Get()->Result; + if (result.IsSuccess()) { + LOG_D("Describe succeeded" + << ": path# " << path.first); + + auto entry = result.GetEntry(); + entry.Name = path.first; // replace by full path + + if (const auto kind = TryTargetKindFromEntryType(entry.Type)) { + LOG_I("Add target" + << ": path# " << path.first + << ", kind# " << kind); + ToAdd.emplace_back(std::move(entry), path.second); + } else { + LOG_W("Unsupported entry type" + << ": path# " << path.first + << ", type# " << entry.Type); + + NYql::TIssues issues; + issues.AddIssue(TStringBuilder() << "Unsupported entry type: " << entry.Type); + Failed.emplace_back(path.first, NYdb::TStatus(NYdb::EStatus::UNSUPPORTED, std::move(issues))); + } + } else { + LOG_E("Describe failed" + << ": path# " << path.first); + + if (IsRetryableError(result)) { + return Retry(*it); + } else { + Failed.emplace_back(path.first, result); + } + } + + Pending.erase(it); + if (Pending) { + return; + } + + if (Failed) { + Send(Parent, new TEvPrivate::TEvDiscoveryResult(ReplicationId, std::move(Failed))); + } else { + Send(Parent, new TEvPrivate::TEvDiscoveryResult(ReplicationId, std::move(ToAdd), {})); + } + + PassAway(); + } + + void Retry(ui32 idx) { + if (ToRetry.empty()) { + Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup); + } + + ToRetry.insert(idx); + } + + void Retry() { + for (const ui32 idx : ToRetry) { + DescribePath(idx); + } + + ToRetry.clear(); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::REPLICATION_CONTROLLER_DISCOVERER; + } + + explicit TDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, TVector<std::pair<TString, TString>>&& paths) + : Parent(parent) + , ReplicationId(rid) + , YdbProxy(proxy) + , Paths(std::move(paths)) + , LogPrefix("Discoverer", ReplicationId) + { + } + + void Bootstrap() { + for (ui32 i = 0; i < Paths.size(); ++i) { + DescribePath(i); + } + + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvYdbProxy::TEvDescribePathResponse, Handle); + sFunc(TEvents::TEvWakeup, Retry); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const ui64 ReplicationId; + const TActorId YdbProxy; + const TVector<std::pair<TString, TString>> Paths; + const TActorLogPrefix LogPrefix; + + THashSet<ui32> Pending; + THashSet<ui32> ToRetry; + TVector<TEvPrivate::TEvDiscoveryResult::TAddEntry> ToAdd; + TVector<TEvPrivate::TEvDiscoveryResult::TFailedEntry> Failed; + +}; // TDiscoverer + +IActor* CreateDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, + TVector<std::pair<TString, TString>>&& specificPaths) +{ + return new TDiscoverer(parent, rid, proxy, std::move(specificPaths)); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/discoverer.h b/ydb/core/tx/replication/controller/discoverer.h index eb989603f5e..235b98c6270 100644 --- a/ydb/core/tx/replication/controller/discoverer.h +++ b/ydb/core/tx/replication/controller/discoverer.h @@ -1,16 +1,16 @@ -#pragma once - -#include <ydb/core/base/defs.h> - -#include <util/generic/vector.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -IActor* CreateDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, - TVector<std::pair<TString, TString>>&& specificPaths); - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include <ydb/core/base/defs.h> + +#include <util/generic/vector.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +IActor* CreateDiscoverer(const TActorId& parent, ui64 rid, const TActorId& proxy, + TVector<std::pair<TString, TString>>&& specificPaths); + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/dst_creator.cpp b/ydb/core/tx/replication/controller/dst_creator.cpp index 356406a80d9..3095dee857b 100644 --- a/ydb/core/tx/replication/controller/dst_creator.cpp +++ b/ydb/core/tx/replication/controller/dst_creator.cpp @@ -1,425 +1,425 @@ -#include "dst_creator.h" -#include "logging.h" -#include "private_events.h" -#include "util.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <ydb/core/base/tablet_pipecache.h> -#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> -#include <ydb/core/tx/schemeshard/schemeshard.h> -#include <ydb/core/tx/tx_proxy/proxy.h> -#include <ydb/core/ydb_convert/table_description.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -using namespace NSchemeShard; - -class TDstCreator: public TActorBootstrapped<TDstCreator> { - void DescribeSrcPath() { - switch (Kind) { - case TReplication::ETargetKind::Table: - Send(YdbProxy, new TEvYdbProxy::TEvDescribeTableRequest(SrcPath, {})); - break; - } - - Become(&TThis::StateDescribeSrcPath); - } - - STATEFN(StateDescribeSrcPath) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvYdbProxy::TEvDescribeTableResponse, Handle); - sFunc(TEvents::TEvWakeup, DescribeSrcPath); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvYdbProxy::TEvDescribeTableResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - - Y_VERIFY(Kind == TReplication::ETargetKind::Table); - const auto& result = ev->Get()->Result; - - if (!result.IsSuccess()) { - if (IsRetryableError(result)) { - return Retry(); - } - - return Error(NKikimrScheme::StatusNotAvailable, TStringBuilder() << "Cannot describe table" - << ": status: " << result.GetStatus() - << ", issue: " << result.GetIssues().ToOneLineString()); - } - - Ydb::Table::CreateTableRequest scheme; - result.GetTableDescription().SerializeTo(scheme); - - Ydb::StatusIds::StatusCode status; - TString error; - if (!FillTableDescription(TxBody, scheme, status, error)) { - return Error(NKikimrScheme::StatusSchemeError, error); - } - - TxBody.MutableCreateTable()->SetName(ToString(ExtractBase(DstPath))); - AllocateTxId(); - } - - void AllocateTxId() { - Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); - Become(&TThis::StateAllocateTxId); - } - - STATEFN(StateAllocateTxId) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - - TxId = ev->Get()->TxId; - PipeCache = ev->Get()->Services.LeaderPipeCache; - CreateDst(); - } - - void CreateDst() { - auto ev = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(TxId, SchemeShardId); - *ev->Record.AddTransaction() = TxBody; - - Send(PipeCache, new TEvPipeCache::TEvForward(ev.Release(), SchemeShardId, true)); - Become(&TThis::StateCreateDst); - } - - STATEFN(StateCreateDst) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvSchemeShard::TEvModifySchemeTransactionResult, Handle); - hFunc(TEvSchemeShard::TEvNotifyTxCompletionResult, Handle); - sFunc(TEvents::TEvWakeup, AllocateTxId); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void SubscribeTx(ui64 txId) { - LOG_D("Subscribe tx" - << ": txId# " << txId); - Send(PipeCache, new TEvPipeCache::TEvForward(new TEvSchemeShard::TEvNotifyTxCompletion(txId), SchemeShardId)); - } - - void Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - const auto& record = ev->Get()->Record; - - switch (record.GetStatus()) { - case NKikimrScheme::StatusAccepted: - DstPathId = TPathId(SchemeShardId, record.GetPathId()); - Y_VERIFY_DEBUG(TxId == record.GetTxId()); - return SubscribeTx(record.GetTxId()); - case NKikimrScheme::StatusMultipleModifications: - if (record.HasPathCreateTxId()) { - NeedToCheck = true; - return SubscribeTx(record.GetPathCreateTxId()); - } else { - return Error(record.GetStatus(), record.GetReason()); - } - break; - case NKikimrScheme::StatusAlreadyExists: - return DescribeDstPath(); - default: - return Error(record.GetStatus(), record.GetReason()); - } - } - - void Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - - if (NeedToCheck) { - DescribeDstPath(); - } else { - Success(); - } - } - - void DescribeDstPath() { - Send(PipeCache, new TEvPipeCache::TEvForward(new TEvSchemeShard::TEvDescribeScheme(DstPath), SchemeShardId)); - Become(&TThis::StateDescribeDstPath); - } - - STATEFN(StateDescribeDstPath) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvSchemeShard::TEvDescribeSchemeResult, Handle); - sFunc(TEvents::TEvWakeup, DescribeDstPath); - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - - void Handle(TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - const auto& record = ev->Get()->GetRecord(); - - switch (record.GetStatus()) { - case NKikimrScheme::StatusSuccess: { - TString error; - if (!CheckScheme(record.GetPathDescription(), error)) { - return Error(NKikimrScheme::StatusSchemeError, error); - } else { - DstPathId = TPathId(record.GetPathOwnerId(), record.GetPathId()); - return Success(); - } - break; - } - case NKikimrScheme::StatusPathDoesNotExist: - return AllocateTxId(); - case NKikimrScheme::StatusSchemeError: - case NKikimrScheme::StatusAccessDenied: - case NKikimrScheme::StatusRedirectDomain: - case NKikimrScheme::StatusNameConflict: - case NKikimrScheme::StatusInvalidParameter: - case NKikimrScheme::StatusPreconditionFailed: - return Error(record.GetStatus(), record.GetReason()); - default: - return Retry(); - } - } - - bool CheckScheme(const NKikimrSchemeOp::TPathDescription& desc, TString& error) const { - switch (Kind) { - case TReplication::ETargetKind::Table: - return CheckTableScheme(desc.GetTable(), error); - } - } - - bool CheckTableScheme(const NKikimrSchemeOp::TTableDescription& got, TString& error) const { - const auto& expected = TxBody.GetCreateTable(); - - // check key - if (expected.KeyColumnNamesSize() != got.KeyColumnNamesSize()) { - error = TStringBuilder() << "Key columns size mismatch" - << ": expected: " << expected.KeyColumnNamesSize() - << ", got: " << got.KeyColumnNamesSize(); - return false; - } - - for (ui32 i = 0; i < expected.KeyColumnNamesSize(); ++i) { - if (expected.GetKeyColumnNames(i) != got.GetKeyColumnNames(i)) { - error = TStringBuilder() << "Key column name mismatch" - << ": position: " << i - << ", expected: " << expected.GetKeyColumnNames(i) - << ", got: " << got.GetKeyColumnNames(i); - return false; - } - } - - // check columns - THashMap<TStringBuf, TStringBuf> columns; - for (const auto& column : got.GetColumns()) { - columns.emplace(column.GetName(), column.GetType()); - } - - if (expected.ColumnsSize() != columns.size()) { - error = TStringBuilder() << "Columns size mismatch" - << ": expected: " << expected.ColumnsSize() - << ", got: " << columns.size(); - return false; - } - - for (const auto& column : expected.GetColumns()) { - auto it = columns.find(column.GetName()); - if (it == columns.end()) { - error = TStringBuilder() << "Cannot find column" - << ": name: " << column.GetName(); - return false; - } - - if (column.GetType() != it->second) { - error = TStringBuilder() << "Column type mismatch" - << ": name: " << column.GetName() - << ", expected: " << column.GetType() - << ", got: " << it->second; - return false; - } - } - - // check indexes - THashMap<TStringBuf, const NKikimrSchemeOp::TIndexDescription*> indexes; - for (const auto& index : got.GetTableIndexes()) { - indexes.emplace(index.GetName(), &index); - } - - if (expected.TableIndexesSize() != indexes.size()) { - error = TStringBuilder() << "Indexes size mismatch" - << ": expected: " << expected.TableIndexesSize() - << ", got: " << indexes.size(); - return false; - } - - for (const auto& index : expected.GetTableIndexes()) { - auto it = indexes.find(index.GetName()); - if (it == indexes.end()) { - error = TStringBuilder() << "Cannot find index" - << ": name: " << index.GetName(); - return false; - } - - if (index.GetType() != it->second->GetType()) { - error = TStringBuilder() << "Index type mismatch" - << ": name: " << index.GetName() - << ", expected: " << NKikimrSchemeOp::EIndexType_Name(index.GetType()) - << ", got: " << NKikimrSchemeOp::EIndexType_Name(it->second->GetType()); - return false; - } - - if (index.KeyColumnNamesSize() != it->second->KeyColumnNamesSize()) { - error = TStringBuilder() << "Index key columns size mismatch" - << ": name: " << index.GetName() - << ", expected: " << index.KeyColumnNamesSize() - << ", got: " << it->second->KeyColumnNamesSize(); - return false; - } - - for (ui32 i = 0; i < index.KeyColumnNamesSize(); ++i) { - if (index.GetKeyColumnNames(i) != it->second->GetKeyColumnNames(i)) { - error = TStringBuilder() << "Index key column name mismatch" - << ": name: " << index.GetName() - << ", position: " << i - << ", expected: " << index.GetKeyColumnNames(i) - << ", got: " << it->second->GetKeyColumnNames(i); - return false; - } - } - - if (index.DataColumnNamesSize() != it->second->DataColumnNamesSize()) { - error = TStringBuilder() << "Index data columns size mismatch" - << ": name: " << index.GetName() - << ", expected: " << index.DataColumnNamesSize() - << ", got: " << it->second->DataColumnNamesSize(); - return false; - } - - for (ui32 i = 0; i < index.DataColumnNamesSize(); ++i) { - if (index.GetDataColumnNames(i) != it->second->GetDataColumnNames(i)) { - error = TStringBuilder() << "Index data column name mismatch" - << ": name: " << index.GetName() - << ", position: " << i - << ", expected: " << index.GetDataColumnNames(i) - << ", got: " << it->second->GetDataColumnNames(i); - return false; - } - } - } - - return true; - } - - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - - if (SchemeShardId == ev->Get()->TabletId) { - return; - } - - Retry(); - } - - void Handle(TEvents::TEvUndelivered::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); - Retry(); - } - - void Success() { - Y_VERIFY(DstPathId); - LOG_I("Success" - << ": dstPathId# " << DstPathId); - - Send(Parent, new TEvPrivate::TEvCreateDstResult(ReplicationId, TargetId, DstPathId)); - PassAway(); - } - - void Error(NKikimrScheme::EStatus status, const TString& error) { - LOG_E("Error" - << ": status# " << status - << ", reason# " << error); - - Send(Parent, new TEvPrivate::TEvCreateDstResult(ReplicationId, TargetId, status, error)); - PassAway(); - } - - void Retry() { - LOG_D("Retry"); - Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::REPLICATION_CONTROLLER_DST_CREATOR; - } - - explicit TDstCreator( - const TActorId& parent, - ui64 schemeShardId, - const TActorId& proxy, - ui64 rid, - ui64 tid, - TReplication::ETargetKind kind, - const TString& srcPath, - const TString& dstPath) - : Parent(parent) - , SchemeShardId(schemeShardId) - , YdbProxy(proxy) - , ReplicationId(rid) - , TargetId(tid) - , Kind(kind) - , SrcPath(srcPath) - , DstPath(dstPath) - , LogPrefix("DstCreator", ReplicationId, TargetId) - { - TxBody.SetWorkingDir(ToString(ExtractParent(DstPath))); - } - - void Bootstrap() { - DescribeSrcPath(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - hFunc(TEvents::TEvUndelivered, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const ui64 SchemeShardId; - const TActorId YdbProxy; - const ui64 ReplicationId; - const ui64 TargetId; - const TReplication::ETargetKind Kind; - const TString SrcPath; - const TString DstPath; - const TActorLogPrefix LogPrefix; - - ui64 TxId = 0; - NKikimrSchemeOp::TModifyScheme TxBody; - TActorId PipeCache; - bool NeedToCheck = false; - TPathId DstPathId; - -}; // TDstCreator - -IActor* CreateDstCreator(const TActorId& parent, ui64 schemeShardId, const TActorId& proxy, - ui64 rid, ui64 tid, TReplication::ETargetKind kind, const TString& srcPath, const TString& dstPath) -{ - return new TDstCreator(parent, schemeShardId, proxy, rid, tid, kind, srcPath, dstPath); -} - -} // NController -} // NReplication -} // NKikimr +#include "dst_creator.h" +#include "logging.h" +#include "private_events.h" +#include "util.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/core/base/tablet_pipecache.h> +#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/core/tx/schemeshard/schemeshard.h> +#include <ydb/core/tx/tx_proxy/proxy.h> +#include <ydb/core/ydb_convert/table_description.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +using namespace NSchemeShard; + +class TDstCreator: public TActorBootstrapped<TDstCreator> { + void DescribeSrcPath() { + switch (Kind) { + case TReplication::ETargetKind::Table: + Send(YdbProxy, new TEvYdbProxy::TEvDescribeTableRequest(SrcPath, {})); + break; + } + + Become(&TThis::StateDescribeSrcPath); + } + + STATEFN(StateDescribeSrcPath) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvYdbProxy::TEvDescribeTableResponse, Handle); + sFunc(TEvents::TEvWakeup, DescribeSrcPath); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvYdbProxy::TEvDescribeTableResponse::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + + Y_VERIFY(Kind == TReplication::ETargetKind::Table); + const auto& result = ev->Get()->Result; + + if (!result.IsSuccess()) { + if (IsRetryableError(result)) { + return Retry(); + } + + return Error(NKikimrScheme::StatusNotAvailable, TStringBuilder() << "Cannot describe table" + << ": status: " << result.GetStatus() + << ", issue: " << result.GetIssues().ToOneLineString()); + } + + Ydb::Table::CreateTableRequest scheme; + result.GetTableDescription().SerializeTo(scheme); + + Ydb::StatusIds::StatusCode status; + TString error; + if (!FillTableDescription(TxBody, scheme, status, error)) { + return Error(NKikimrScheme::StatusSchemeError, error); + } + + TxBody.MutableCreateTable()->SetName(ToString(ExtractBase(DstPath))); + AllocateTxId(); + } + + void AllocateTxId() { + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId); + Become(&TThis::StateAllocateTxId); + } + + STATEFN(StateAllocateTxId) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + + TxId = ev->Get()->TxId; + PipeCache = ev->Get()->Services.LeaderPipeCache; + CreateDst(); + } + + void CreateDst() { + auto ev = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(TxId, SchemeShardId); + *ev->Record.AddTransaction() = TxBody; + + Send(PipeCache, new TEvPipeCache::TEvForward(ev.Release(), SchemeShardId, true)); + Become(&TThis::StateCreateDst); + } + + STATEFN(StateCreateDst) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvSchemeShard::TEvModifySchemeTransactionResult, Handle); + hFunc(TEvSchemeShard::TEvNotifyTxCompletionResult, Handle); + sFunc(TEvents::TEvWakeup, AllocateTxId); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void SubscribeTx(ui64 txId) { + LOG_D("Subscribe tx" + << ": txId# " << txId); + Send(PipeCache, new TEvPipeCache::TEvForward(new TEvSchemeShard::TEvNotifyTxCompletion(txId), SchemeShardId)); + } + + void Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + const auto& record = ev->Get()->Record; + + switch (record.GetStatus()) { + case NKikimrScheme::StatusAccepted: + DstPathId = TPathId(SchemeShardId, record.GetPathId()); + Y_VERIFY_DEBUG(TxId == record.GetTxId()); + return SubscribeTx(record.GetTxId()); + case NKikimrScheme::StatusMultipleModifications: + if (record.HasPathCreateTxId()) { + NeedToCheck = true; + return SubscribeTx(record.GetPathCreateTxId()); + } else { + return Error(record.GetStatus(), record.GetReason()); + } + break; + case NKikimrScheme::StatusAlreadyExists: + return DescribeDstPath(); + default: + return Error(record.GetStatus(), record.GetReason()); + } + } + + void Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + + if (NeedToCheck) { + DescribeDstPath(); + } else { + Success(); + } + } + + void DescribeDstPath() { + Send(PipeCache, new TEvPipeCache::TEvForward(new TEvSchemeShard::TEvDescribeScheme(DstPath), SchemeShardId)); + Become(&TThis::StateDescribeDstPath); + } + + STATEFN(StateDescribeDstPath) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvSchemeShard::TEvDescribeSchemeResult, Handle); + sFunc(TEvents::TEvWakeup, DescribeDstPath); + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + + void Handle(TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + const auto& record = ev->Get()->GetRecord(); + + switch (record.GetStatus()) { + case NKikimrScheme::StatusSuccess: { + TString error; + if (!CheckScheme(record.GetPathDescription(), error)) { + return Error(NKikimrScheme::StatusSchemeError, error); + } else { + DstPathId = TPathId(record.GetPathOwnerId(), record.GetPathId()); + return Success(); + } + break; + } + case NKikimrScheme::StatusPathDoesNotExist: + return AllocateTxId(); + case NKikimrScheme::StatusSchemeError: + case NKikimrScheme::StatusAccessDenied: + case NKikimrScheme::StatusRedirectDomain: + case NKikimrScheme::StatusNameConflict: + case NKikimrScheme::StatusInvalidParameter: + case NKikimrScheme::StatusPreconditionFailed: + return Error(record.GetStatus(), record.GetReason()); + default: + return Retry(); + } + } + + bool CheckScheme(const NKikimrSchemeOp::TPathDescription& desc, TString& error) const { + switch (Kind) { + case TReplication::ETargetKind::Table: + return CheckTableScheme(desc.GetTable(), error); + } + } + + bool CheckTableScheme(const NKikimrSchemeOp::TTableDescription& got, TString& error) const { + const auto& expected = TxBody.GetCreateTable(); + + // check key + if (expected.KeyColumnNamesSize() != got.KeyColumnNamesSize()) { + error = TStringBuilder() << "Key columns size mismatch" + << ": expected: " << expected.KeyColumnNamesSize() + << ", got: " << got.KeyColumnNamesSize(); + return false; + } + + for (ui32 i = 0; i < expected.KeyColumnNamesSize(); ++i) { + if (expected.GetKeyColumnNames(i) != got.GetKeyColumnNames(i)) { + error = TStringBuilder() << "Key column name mismatch" + << ": position: " << i + << ", expected: " << expected.GetKeyColumnNames(i) + << ", got: " << got.GetKeyColumnNames(i); + return false; + } + } + + // check columns + THashMap<TStringBuf, TStringBuf> columns; + for (const auto& column : got.GetColumns()) { + columns.emplace(column.GetName(), column.GetType()); + } + + if (expected.ColumnsSize() != columns.size()) { + error = TStringBuilder() << "Columns size mismatch" + << ": expected: " << expected.ColumnsSize() + << ", got: " << columns.size(); + return false; + } + + for (const auto& column : expected.GetColumns()) { + auto it = columns.find(column.GetName()); + if (it == columns.end()) { + error = TStringBuilder() << "Cannot find column" + << ": name: " << column.GetName(); + return false; + } + + if (column.GetType() != it->second) { + error = TStringBuilder() << "Column type mismatch" + << ": name: " << column.GetName() + << ", expected: " << column.GetType() + << ", got: " << it->second; + return false; + } + } + + // check indexes + THashMap<TStringBuf, const NKikimrSchemeOp::TIndexDescription*> indexes; + for (const auto& index : got.GetTableIndexes()) { + indexes.emplace(index.GetName(), &index); + } + + if (expected.TableIndexesSize() != indexes.size()) { + error = TStringBuilder() << "Indexes size mismatch" + << ": expected: " << expected.TableIndexesSize() + << ", got: " << indexes.size(); + return false; + } + + for (const auto& index : expected.GetTableIndexes()) { + auto it = indexes.find(index.GetName()); + if (it == indexes.end()) { + error = TStringBuilder() << "Cannot find index" + << ": name: " << index.GetName(); + return false; + } + + if (index.GetType() != it->second->GetType()) { + error = TStringBuilder() << "Index type mismatch" + << ": name: " << index.GetName() + << ", expected: " << NKikimrSchemeOp::EIndexType_Name(index.GetType()) + << ", got: " << NKikimrSchemeOp::EIndexType_Name(it->second->GetType()); + return false; + } + + if (index.KeyColumnNamesSize() != it->second->KeyColumnNamesSize()) { + error = TStringBuilder() << "Index key columns size mismatch" + << ": name: " << index.GetName() + << ", expected: " << index.KeyColumnNamesSize() + << ", got: " << it->second->KeyColumnNamesSize(); + return false; + } + + for (ui32 i = 0; i < index.KeyColumnNamesSize(); ++i) { + if (index.GetKeyColumnNames(i) != it->second->GetKeyColumnNames(i)) { + error = TStringBuilder() << "Index key column name mismatch" + << ": name: " << index.GetName() + << ", position: " << i + << ", expected: " << index.GetKeyColumnNames(i) + << ", got: " << it->second->GetKeyColumnNames(i); + return false; + } + } + + if (index.DataColumnNamesSize() != it->second->DataColumnNamesSize()) { + error = TStringBuilder() << "Index data columns size mismatch" + << ": name: " << index.GetName() + << ", expected: " << index.DataColumnNamesSize() + << ", got: " << it->second->DataColumnNamesSize(); + return false; + } + + for (ui32 i = 0; i < index.DataColumnNamesSize(); ++i) { + if (index.GetDataColumnNames(i) != it->second->GetDataColumnNames(i)) { + error = TStringBuilder() << "Index data column name mismatch" + << ": name: " << index.GetName() + << ", position: " << i + << ", expected: " << index.GetDataColumnNames(i) + << ", got: " << it->second->GetDataColumnNames(i); + return false; + } + } + } + + return true; + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + + if (SchemeShardId == ev->Get()->TabletId) { + return; + } + + Retry(); + } + + void Handle(TEvents::TEvUndelivered::TPtr& ev) { + LOG_T("Handle " << ev->Get()->ToString()); + Retry(); + } + + void Success() { + Y_VERIFY(DstPathId); + LOG_I("Success" + << ": dstPathId# " << DstPathId); + + Send(Parent, new TEvPrivate::TEvCreateDstResult(ReplicationId, TargetId, DstPathId)); + PassAway(); + } + + void Error(NKikimrScheme::EStatus status, const TString& error) { + LOG_E("Error" + << ": status# " << status + << ", reason# " << error); + + Send(Parent, new TEvPrivate::TEvCreateDstResult(ReplicationId, TargetId, status, error)); + PassAway(); + } + + void Retry() { + LOG_D("Retry"); + Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::REPLICATION_CONTROLLER_DST_CREATOR; + } + + explicit TDstCreator( + const TActorId& parent, + ui64 schemeShardId, + const TActorId& proxy, + ui64 rid, + ui64 tid, + TReplication::ETargetKind kind, + const TString& srcPath, + const TString& dstPath) + : Parent(parent) + , SchemeShardId(schemeShardId) + , YdbProxy(proxy) + , ReplicationId(rid) + , TargetId(tid) + , Kind(kind) + , SrcPath(srcPath) + , DstPath(dstPath) + , LogPrefix("DstCreator", ReplicationId, TargetId) + { + TxBody.SetWorkingDir(ToString(ExtractParent(DstPath))); + } + + void Bootstrap() { + DescribeSrcPath(); + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(TEvents::TEvUndelivered, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const ui64 SchemeShardId; + const TActorId YdbProxy; + const ui64 ReplicationId; + const ui64 TargetId; + const TReplication::ETargetKind Kind; + const TString SrcPath; + const TString DstPath; + const TActorLogPrefix LogPrefix; + + ui64 TxId = 0; + NKikimrSchemeOp::TModifyScheme TxBody; + TActorId PipeCache; + bool NeedToCheck = false; + TPathId DstPathId; + +}; // TDstCreator + +IActor* CreateDstCreator(const TActorId& parent, ui64 schemeShardId, const TActorId& proxy, + ui64 rid, ui64 tid, TReplication::ETargetKind kind, const TString& srcPath, const TString& dstPath) +{ + return new TDstCreator(parent, schemeShardId, proxy, rid, tid, kind, srcPath, dstPath); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/dst_creator.h b/ydb/core/tx/replication/controller/dst_creator.h index b98a7b01ac7..7863ea8ed59 100644 --- a/ydb/core/tx/replication/controller/dst_creator.h +++ b/ydb/core/tx/replication/controller/dst_creator.h @@ -1,16 +1,16 @@ -#pragma once - -#include "replication.h" - -#include <ydb/core/base/defs.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -IActor* CreateDstCreator(const TActorId& parent, ui64 schemeShardId, const TActorId& proxy, - ui64 rid, ui64 tid, TReplication::ETargetKind kind, const TString& srcPath, const TString& dstPath); - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include "replication.h" + +#include <ydb/core/base/defs.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +IActor* CreateDstCreator(const TActorId& parent, ui64 schemeShardId, const TActorId& proxy, + ui64 rid, ui64 tid, TReplication::ETargetKind kind, const TString& srcPath, const TString& dstPath); + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/logging.cpp b/ydb/core/tx/replication/controller/logging.cpp index 5119e110443..4986d55076c 100644 --- a/ydb/core/tx/replication/controller/logging.cpp +++ b/ydb/core/tx/replication/controller/logging.cpp @@ -1,56 +1,56 @@ -#include "logging.h" - -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -TTabletLogPrefix::TTabletLogPrefix(const TController* self) - : TabletId(self->TabletID()) -{ -} - -TTabletLogPrefix::TTabletLogPrefix(const TController* self, const TString& txName) - : TabletId(self->TabletID()) - , TxName(txName) -{ -} - -void TTabletLogPrefix::Out(IOutputStream& output) const { - output << "[controller " << TabletId << "]"; - if (TxName) { - output << "[" << TxName << "]"; - } - output << " "; -} - -TActorLogPrefix::TActorLogPrefix(const TString& activity, ui64 rid, ui64 tid) - : Activity(activity) - , ReplicationId(rid) - , TargetId(tid) -{ -} - -void TActorLogPrefix::Out(IOutputStream& output) const { - output << "[" << Activity << "]"; - if (ReplicationId) { - output << "[rid " << ReplicationId << "]"; - } - if (TargetId) { - output << "[tid " << TargetId << "]"; - } - output << " "; -} - -} // NController -} // NReplication -} // NKikimr - -Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TTabletLogPrefix, output, value) { - value.Out(output); -} - -Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TActorLogPrefix, output, value) { - value.Out(output); -} +#include "logging.h" + +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +TTabletLogPrefix::TTabletLogPrefix(const TController* self) + : TabletId(self->TabletID()) +{ +} + +TTabletLogPrefix::TTabletLogPrefix(const TController* self, const TString& txName) + : TabletId(self->TabletID()) + , TxName(txName) +{ +} + +void TTabletLogPrefix::Out(IOutputStream& output) const { + output << "[controller " << TabletId << "]"; + if (TxName) { + output << "[" << TxName << "]"; + } + output << " "; +} + +TActorLogPrefix::TActorLogPrefix(const TString& activity, ui64 rid, ui64 tid) + : Activity(activity) + , ReplicationId(rid) + , TargetId(tid) +{ +} + +void TActorLogPrefix::Out(IOutputStream& output) const { + output << "[" << Activity << "]"; + if (ReplicationId) { + output << "[rid " << ReplicationId << "]"; + } + if (TargetId) { + output << "[tid " << TargetId << "]"; + } + output << " "; +} + +} // NController +} // NReplication +} // NKikimr + +Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TTabletLogPrefix, output, value) { + value.Out(output); +} + +Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TActorLogPrefix, output, value) { + value.Out(output); +} diff --git a/ydb/core/tx/replication/controller/logging.h b/ydb/core/tx/replication/controller/logging.h index 374786d54ef..63e3bf23c25 100644 --- a/ydb/core/tx/replication/controller/logging.h +++ b/ydb/core/tx/replication/controller/logging.h @@ -1,54 +1,54 @@ -#pragma once - -#include <ydb/core/base/defs.h> - -#include <library/cpp/actors/core/log.h> - -#define CLOG_T(ctx, stream) LOG_TRACE_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) -#define CLOG_D(ctx, stream) LOG_DEBUG_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) -#define CLOG_I(ctx, stream) LOG_INFO_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) -#define CLOG_N(ctx, stream) LOG_NOTICE_S(ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) -#define CLOG_W(ctx, stream) LOG_WARN_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) -#define CLOG_E(ctx, stream) LOG_ERROR_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) - -#define LOG_T(stream) CLOG_T(*TlsActivationContext, stream) -#define LOG_D(stream) CLOG_D(*TlsActivationContext, stream) -#define LOG_I(stream) CLOG_I(*TlsActivationContext, stream) -#define LOG_N(stream) CLOG_N(*TlsActivationContext, stream) -#define LOG_W(stream) CLOG_W(*TlsActivationContext, stream) -#define LOG_E(stream) CLOG_E(*TlsActivationContext, stream) - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController; - -class TTabletLogPrefix { -public: - explicit TTabletLogPrefix(const TController* self); - explicit TTabletLogPrefix(const TController* self, const TString& txName); - - void Out(IOutputStream& out) const; - -private: - const ui64 TabletId; - const TString TxName; -}; - -class TActorLogPrefix { -public: - TActorLogPrefix() = default; - explicit TActorLogPrefix(const TString& activity, ui64 rid = 0, ui64 tid = 0); - - void Out(IOutputStream& out) const; - -private: - const TString Activity; - const ui64 ReplicationId = 0; - const ui64 TargetId = 0; -}; - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include <ydb/core/base/defs.h> + +#include <library/cpp/actors/core/log.h> + +#define CLOG_T(ctx, stream) LOG_TRACE_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) +#define CLOG_D(ctx, stream) LOG_DEBUG_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) +#define CLOG_I(ctx, stream) LOG_INFO_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) +#define CLOG_N(ctx, stream) LOG_NOTICE_S(ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) +#define CLOG_W(ctx, stream) LOG_WARN_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) +#define CLOG_E(ctx, stream) LOG_ERROR_S (ctx, NKikimrServices::REPLICATION_CONTROLLER, LogPrefix << stream) + +#define LOG_T(stream) CLOG_T(*TlsActivationContext, stream) +#define LOG_D(stream) CLOG_D(*TlsActivationContext, stream) +#define LOG_I(stream) CLOG_I(*TlsActivationContext, stream) +#define LOG_N(stream) CLOG_N(*TlsActivationContext, stream) +#define LOG_W(stream) CLOG_W(*TlsActivationContext, stream) +#define LOG_E(stream) CLOG_E(*TlsActivationContext, stream) + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController; + +class TTabletLogPrefix { +public: + explicit TTabletLogPrefix(const TController* self); + explicit TTabletLogPrefix(const TController* self, const TString& txName); + + void Out(IOutputStream& out) const; + +private: + const ui64 TabletId; + const TString TxName; +}; + +class TActorLogPrefix { +public: + TActorLogPrefix() = default; + explicit TActorLogPrefix(const TString& activity, ui64 rid = 0, ui64 tid = 0); + + void Out(IOutputStream& out) const; + +private: + const TString Activity; + const ui64 ReplicationId = 0; + const ui64 TargetId = 0; +}; + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/private_events.cpp b/ydb/core/tx/replication/controller/private_events.cpp index 763943b0330..d3c1dcc7f37 100644 --- a/ydb/core/tx/replication/controller/private_events.cpp +++ b/ydb/core/tx/replication/controller/private_events.cpp @@ -1,110 +1,110 @@ -#include "private_events.h" - -#include <util/string/join.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -TEvPrivate::TEvDiscoveryResult::TEvDiscoveryResult(ui64 rid, TVector<TAddEntry>&& toAdd, TVector<ui64>&& toDel) - : ReplicationId(rid) - , ToAdd(std::move(toAdd)) - , ToDelete(std::move(toDel)) -{ -} - -TEvPrivate::TEvDiscoveryResult::TEvDiscoveryResult(ui64 rid, TVector<TFailedEntry>&& failed) - : ReplicationId(rid) - , Failed(std::move(failed)) -{ -} - -TString TEvPrivate::TEvDiscoveryResult::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " ReplicationId: " << ReplicationId - << " ToAdd [" << JoinSeq(",", ToAdd) << "]" - << " ToDelete [" << JoinSeq(",", ToDelete) << "]" - << " Failed [" << JoinSeq(",", Failed) << "]" - << " }"; -} - -bool TEvPrivate::TEvDiscoveryResult::IsSuccess() const { - return Failed.empty(); -} - -TEvPrivate::TEvAssignStreamName::TEvAssignStreamName(ui64 rid, ui64 tid) - : ReplicationId(rid) - , TargetId(tid) -{ -} - -TString TEvPrivate::TEvAssignStreamName::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " ReplicationId: " << ReplicationId - << " TargetId: " << TargetId - << " }"; -} - -TEvPrivate::TEvCreateStreamResult::TEvCreateStreamResult(ui64 rid, ui64 tid, NYdb::TStatus&& status) - : ReplicationId(rid) - , TargetId(tid) - , Status(std::move(status)) -{ -} - -TString TEvPrivate::TEvCreateStreamResult::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " ReplicationId: " << ReplicationId - << " TargetId: " << TargetId - << " Status: " << Status.GetStatus() - << " Issues: " << Status.GetIssues().ToOneLineString() - << " }"; -} - -bool TEvPrivate::TEvCreateStreamResult::IsSuccess() const { - return Status.IsSuccess(); -} - -TEvPrivate::TEvCreateDstResult::TEvCreateDstResult(ui64 rid, ui64 tid, const TPathId& dstPathId) - : ReplicationId(rid) - , TargetId(tid) - , DstPathId(dstPathId) - , Status(NKikimrScheme::StatusSuccess) -{ -} - -TEvPrivate::TEvCreateDstResult::TEvCreateDstResult(ui64 rid, ui64 tid, NKikimrScheme::EStatus status, const TString& error) - : ReplicationId(rid) - , TargetId(tid) - , Status(status) - , Error(error) -{ -} - -TString TEvPrivate::TEvCreateDstResult::ToString() const { - return TStringBuilder() << ToStringHeader() << " {" - << " ReplicationId: " << ReplicationId - << " TargetId: " << TargetId - << " DstPathId: " << DstPathId - << " Status: " << NKikimrScheme::EStatus_Name(Status) - << " Error: " << Error - << " }"; -} - -bool TEvPrivate::TEvCreateDstResult::IsSuccess() const { - return Status == NKikimrScheme::StatusSuccess; -} - -} // NController -} // NReplication -} // NKikimr - -Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TEvPrivate::TEvDiscoveryResult::TAddEntry, stream, value) { - stream << value.first.Name << " (" << value.first.Type << ")"; -} - -Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TEvPrivate::TEvDiscoveryResult::TFailedEntry, stream, value) { - stream << value.first << ": " << value.second.GetStatus() << " ("; - value.second.GetIssues().PrintTo(stream, true); - stream << ")"; -} +#include "private_events.h" + +#include <util/string/join.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +TEvPrivate::TEvDiscoveryResult::TEvDiscoveryResult(ui64 rid, TVector<TAddEntry>&& toAdd, TVector<ui64>&& toDel) + : ReplicationId(rid) + , ToAdd(std::move(toAdd)) + , ToDelete(std::move(toDel)) +{ +} + +TEvPrivate::TEvDiscoveryResult::TEvDiscoveryResult(ui64 rid, TVector<TFailedEntry>&& failed) + : ReplicationId(rid) + , Failed(std::move(failed)) +{ +} + +TString TEvPrivate::TEvDiscoveryResult::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " ReplicationId: " << ReplicationId + << " ToAdd [" << JoinSeq(",", ToAdd) << "]" + << " ToDelete [" << JoinSeq(",", ToDelete) << "]" + << " Failed [" << JoinSeq(",", Failed) << "]" + << " }"; +} + +bool TEvPrivate::TEvDiscoveryResult::IsSuccess() const { + return Failed.empty(); +} + +TEvPrivate::TEvAssignStreamName::TEvAssignStreamName(ui64 rid, ui64 tid) + : ReplicationId(rid) + , TargetId(tid) +{ +} + +TString TEvPrivate::TEvAssignStreamName::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " ReplicationId: " << ReplicationId + << " TargetId: " << TargetId + << " }"; +} + +TEvPrivate::TEvCreateStreamResult::TEvCreateStreamResult(ui64 rid, ui64 tid, NYdb::TStatus&& status) + : ReplicationId(rid) + , TargetId(tid) + , Status(std::move(status)) +{ +} + +TString TEvPrivate::TEvCreateStreamResult::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " ReplicationId: " << ReplicationId + << " TargetId: " << TargetId + << " Status: " << Status.GetStatus() + << " Issues: " << Status.GetIssues().ToOneLineString() + << " }"; +} + +bool TEvPrivate::TEvCreateStreamResult::IsSuccess() const { + return Status.IsSuccess(); +} + +TEvPrivate::TEvCreateDstResult::TEvCreateDstResult(ui64 rid, ui64 tid, const TPathId& dstPathId) + : ReplicationId(rid) + , TargetId(tid) + , DstPathId(dstPathId) + , Status(NKikimrScheme::StatusSuccess) +{ +} + +TEvPrivate::TEvCreateDstResult::TEvCreateDstResult(ui64 rid, ui64 tid, NKikimrScheme::EStatus status, const TString& error) + : ReplicationId(rid) + , TargetId(tid) + , Status(status) + , Error(error) +{ +} + +TString TEvPrivate::TEvCreateDstResult::ToString() const { + return TStringBuilder() << ToStringHeader() << " {" + << " ReplicationId: " << ReplicationId + << " TargetId: " << TargetId + << " DstPathId: " << DstPathId + << " Status: " << NKikimrScheme::EStatus_Name(Status) + << " Error: " << Error + << " }"; +} + +bool TEvPrivate::TEvCreateDstResult::IsSuccess() const { + return Status == NKikimrScheme::StatusSuccess; +} + +} // NController +} // NReplication +} // NKikimr + +Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TEvPrivate::TEvDiscoveryResult::TAddEntry, stream, value) { + stream << value.first.Name << " (" << value.first.Type << ")"; +} + +Y_DECLARE_OUT_SPEC(, NKikimr::NReplication::NController::TEvPrivate::TEvDiscoveryResult::TFailedEntry, stream, value) { + stream << value.first << ": " << value.second.GetStatus() << " ("; + value.second.GetIssues().PrintTo(stream, true); + stream << ")"; +} diff --git a/ydb/core/tx/replication/controller/private_events.h b/ydb/core/tx/replication/controller/private_events.h index fdb04d5f223..a25574b9662 100644 --- a/ydb/core/tx/replication/controller/private_events.h +++ b/ydb/core/tx/replication/controller/private_events.h @@ -1,79 +1,79 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> - -#include <ydb/core/base/defs.h> -#include <ydb/core/base/events.h> -#include <ydb/core/base/pathid.h> -#include <ydb/core/protos/flat_tx_scheme.pb.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -struct TEvPrivate { - enum EEv { - EvDiscoveryResult = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvAssignStreamName, - EvCreateStreamResult, - EvCreateDstResult, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); - - struct TEvDiscoveryResult: public TEventLocal<TEvDiscoveryResult, EvDiscoveryResult> { - using TAddEntry = std::pair<NYdb::NScheme::TSchemeEntry, TString>; // src, dst - using TFailedEntry = std::pair<TString, NYdb::TStatus>; // src, error - - const ui64 ReplicationId; - TVector<TAddEntry> ToAdd; - TVector<ui64> ToDelete; - TVector<TFailedEntry> Failed; - - explicit TEvDiscoveryResult(ui64 rid, TVector<TAddEntry>&& toAdd, TVector<ui64>&& toDel); - explicit TEvDiscoveryResult(ui64 rid, TVector<TFailedEntry>&& failed); - TString ToString() const override; - - bool IsSuccess() const; - }; - - struct TEvAssignStreamName: public TEventLocal<TEvAssignStreamName, EvAssignStreamName> { - const ui64 ReplicationId; - const ui64 TargetId; - - explicit TEvAssignStreamName(ui64 rid, ui64 tid); - TString ToString() const override; - }; - - struct TEvCreateStreamResult: public TEventLocal<TEvCreateStreamResult, EvCreateStreamResult> { - const ui64 ReplicationId; - const ui64 TargetId; - const NYdb::TStatus Status; - - explicit TEvCreateStreamResult(ui64 rid, ui64 tid, NYdb::TStatus&& status); - TString ToString() const override; - - bool IsSuccess() const; - }; - - struct TEvCreateDstResult: public TEventLocal<TEvCreateDstResult, EvCreateDstResult> { - const ui64 ReplicationId; - const ui64 TargetId; - const TPathId DstPathId; - const NKikimrScheme::EStatus Status; - const TString Error; - - explicit TEvCreateDstResult(ui64 rid, ui64 tid, const TPathId& dstPathId); - explicit TEvCreateDstResult(ui64 rid, ui64 tid, NKikimrScheme::EStatus status, const TString& error); - TString ToString() const override; - - bool IsSuccess() const; - }; - -}; // TEvPrivate - -} // NController -} // NReplication -} // NKikimr + +#include <ydb/core/base/defs.h> +#include <ydb/core/base/events.h> +#include <ydb/core/base/pathid.h> +#include <ydb/core/protos/flat_tx_scheme.pb.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +struct TEvPrivate { + enum EEv { + EvDiscoveryResult = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvAssignStreamName, + EvCreateStreamResult, + EvCreateDstResult, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + struct TEvDiscoveryResult: public TEventLocal<TEvDiscoveryResult, EvDiscoveryResult> { + using TAddEntry = std::pair<NYdb::NScheme::TSchemeEntry, TString>; // src, dst + using TFailedEntry = std::pair<TString, NYdb::TStatus>; // src, error + + const ui64 ReplicationId; + TVector<TAddEntry> ToAdd; + TVector<ui64> ToDelete; + TVector<TFailedEntry> Failed; + + explicit TEvDiscoveryResult(ui64 rid, TVector<TAddEntry>&& toAdd, TVector<ui64>&& toDel); + explicit TEvDiscoveryResult(ui64 rid, TVector<TFailedEntry>&& failed); + TString ToString() const override; + + bool IsSuccess() const; + }; + + struct TEvAssignStreamName: public TEventLocal<TEvAssignStreamName, EvAssignStreamName> { + const ui64 ReplicationId; + const ui64 TargetId; + + explicit TEvAssignStreamName(ui64 rid, ui64 tid); + TString ToString() const override; + }; + + struct TEvCreateStreamResult: public TEventLocal<TEvCreateStreamResult, EvCreateStreamResult> { + const ui64 ReplicationId; + const ui64 TargetId; + const NYdb::TStatus Status; + + explicit TEvCreateStreamResult(ui64 rid, ui64 tid, NYdb::TStatus&& status); + TString ToString() const override; + + bool IsSuccess() const; + }; + + struct TEvCreateDstResult: public TEventLocal<TEvCreateDstResult, EvCreateDstResult> { + const ui64 ReplicationId; + const ui64 TargetId; + const TPathId DstPathId; + const NKikimrScheme::EStatus Status; + const TString Error; + + explicit TEvCreateDstResult(ui64 rid, ui64 tid, const TPathId& dstPathId); + explicit TEvCreateDstResult(ui64 rid, ui64 tid, NKikimrScheme::EStatus status, const TString& error); + TString ToString() const override; + + bool IsSuccess() const; + }; + +}; // TEvPrivate + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/public_events.h b/ydb/core/tx/replication/controller/public_events.h index 74b148523b6..ea422a593e5 100644 --- a/ydb/core/tx/replication/controller/public_events.h +++ b/ydb/core/tx/replication/controller/public_events.h @@ -1,44 +1,44 @@ -#pragma once - -#include <ydb/core/base/defs.h> -#include <ydb/core/base/events.h> -#include <ydb/core/protos/replication.pb.h> - -namespace NKikimr { -namespace NReplication { - -struct TEvController { - enum EEv { - EvCreateReplication = EventSpaceBegin(TKikimrEvents::ES_REPLICATION_CONTROLLER), - EvCreateReplicationResult, - EvAlterReplication, - EvAlterReplicationResult, - EvDropReplication, - EvDropReplicationResult, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_REPLICATION_CONTROLLER), - "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_REPLICATION_CONTROLLER)"); - - struct TEvCreateReplication - : public TEventPB<TEvCreateReplication, NKikimrReplication::TEvCreateReplication, EvCreateReplication> - {}; - - struct TEvCreateReplicationResult - : public TEventPB<TEvCreateReplicationResult, NKikimrReplication::TEvCreateReplicationResult, EvCreateReplicationResult> - {}; - - struct TEvDropReplication - : public TEventPB<TEvDropReplication, NKikimrReplication::TEvDropReplication, EvDropReplication> - {}; - - struct TEvDropReplicationResult - : public TEventPB<TEvDropReplicationResult, NKikimrReplication::TEvDropReplicationResult, EvDropReplicationResult> - {}; - -}; // TEvController - -} // NReplication -} // NKikimr +#pragma once + +#include <ydb/core/base/defs.h> +#include <ydb/core/base/events.h> +#include <ydb/core/protos/replication.pb.h> + +namespace NKikimr { +namespace NReplication { + +struct TEvController { + enum EEv { + EvCreateReplication = EventSpaceBegin(TKikimrEvents::ES_REPLICATION_CONTROLLER), + EvCreateReplicationResult, + EvAlterReplication, + EvAlterReplicationResult, + EvDropReplication, + EvDropReplicationResult, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_REPLICATION_CONTROLLER), + "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_REPLICATION_CONTROLLER)"); + + struct TEvCreateReplication + : public TEventPB<TEvCreateReplication, NKikimrReplication::TEvCreateReplication, EvCreateReplication> + {}; + + struct TEvCreateReplicationResult + : public TEventPB<TEvCreateReplicationResult, NKikimrReplication::TEvCreateReplicationResult, EvCreateReplicationResult> + {}; + + struct TEvDropReplication + : public TEventPB<TEvDropReplication, NKikimrReplication::TEvDropReplication, EvDropReplication> + {}; + + struct TEvDropReplicationResult + : public TEventPB<TEvDropReplicationResult, NKikimrReplication::TEvDropReplicationResult, EvDropReplicationResult> + {}; + +}; // TEvController + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/replication.cpp b/ydb/core/tx/replication/controller/replication.cpp index c590ec687f4..aa5c26c151d 100644 --- a/ydb/core/tx/replication/controller/replication.cpp +++ b/ydb/core/tx/replication/controller/replication.cpp @@ -1,230 +1,230 @@ -#include "discoverer.h" -#include "replication.h" -#include "target_table.h" -#include "util.h" - -#include <ydb/core/protos/replication.pb.h> -#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> -#include <ydb/core/util/yverify_stream.h> - -#include <library/cpp/actors/core/events.h> - -#include <util/generic/hash.h> -#include <util/generic/ptr.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TReplication::TImpl { - friend class TReplication; - - template <typename... Args> - ITarget* CreateTarget(ui64 id, ETargetKind kind, Args&&... args) const { - switch (kind) { - case ETargetKind::Table: - return new TTableTarget(ReplicationId, id, std::forward<Args>(args)...); - } - } - - void DiscoverTargets(const TActorContext& ctx) { - switch (Config.GetTargetCase()) { - case NKikimrReplication::TReplicationConfig::kEverything: - return ErrorState("Not implemented"); - - case NKikimrReplication::TReplicationConfig::kSpecific: { - TVector<std::pair<TString, TString>> paths; - for (const auto& target : Config.GetSpecific().GetTargets()) { - paths.emplace_back(target.GetSrcPath(), target.GetDstPath()); - } - - Discoverer = ctx.Register(CreateDiscoverer(ctx.SelfID, ReplicationId, YdbProxy, std::move(paths))); - break; - } - - default: - return ErrorState(TStringBuilder() << "Unexpected targets: " << Config.GetTargetCase()); - } - } - - void ProgressTargets(const TActorContext& ctx) { - for (auto& [_, target] : Targets) { - target->Progress(PathId.OwnerId, YdbProxy, ctx); - } - } - -public: - template <typename T> - explicit TImpl(ui64 id, const TPathId& pathId, T&& config) - : ReplicationId(id) - , PathId(pathId) - , Config(std::forward<T>(config)) - { - } - - template <typename... Args> - ui64 AddTarget(ui64 id, ETargetKind kind, Args&&... args) { - const auto res = Targets.emplace(id, CreateTarget(id, kind, std::forward<Args>(args)...)); - Y_VERIFY_S(res.second, "Duplicate target: " << id); - return id; - } - - template <typename... Args> - ui64 AddTarget(ETargetKind kind, Args&&... args) { - return AddTarget(NextTargetId++, kind, std::forward<Args>(args)...); - } - - ITarget* FindTarget(ui64 id) { - auto it = Targets.find(id); - return it != Targets.end() - ? it->second.Get() - : nullptr; - } - - void Progress(const TActorContext& ctx) { - if (!YdbProxy) { - THolder<IActor> ydbProxy; - switch (Config.GetCredentialsCase()) { - case NKikimrReplication::TReplicationConfig::kStaticCredentials: - ydbProxy.Reset(CreateYdbProxy(Config.GetSrcEndpoint(), Config.GetSrcDatabase(), Config.GetStaticCredentials())); - break; - default: - ErrorState(TStringBuilder() << "Unexpected credentials: " << Config.GetCredentialsCase()); - break; - } - - if (ydbProxy) { - YdbProxy = ctx.Register(ydbProxy.Release()); - } - } - - switch (State) { - case EState::Ready: - if (!Targets) { - return DiscoverTargets(ctx); - } else { - return ProgressTargets(ctx); - } - case EState::Removing: - return; // TODO - case EState::Error: - return; - } - } - - void Shutdown(const TActorContext& ctx) { - for (auto& [_, target] : Targets) { - target->Shutdown(ctx); - } - - for (auto& x : TVector<TActorId>{Discoverer, YdbProxy}) { - if (auto actorId = std::exchange(x, {})) { - ctx.Send(actorId, new TEvents::TEvPoison()); - } - } - } - - void SetState(EState state, TString issue = {}) { - State = state; - Issue = TruncatedIssue(issue); - } - - void ErrorState(TString issue) { - SetState(EState::Error, issue); - } - -private: - const ui64 ReplicationId; - const TPathId PathId; - - NKikimrReplication::TReplicationConfig Config; - EState State = EState::Ready; - TString Issue; - ui64 NextTargetId = 1; - THashMap<ui64, THolder<ITarget>> Targets; - TActorId YdbProxy; - TActorId Discoverer; - -}; // TImpl - -TReplication::TReplication(ui64 id, const TPathId& pathId, const NKikimrReplication::TReplicationConfig& config) - : Impl(std::make_shared<TImpl>(id, pathId, config)) -{ -} - -TReplication::TReplication(ui64 id, const TPathId& pathId, NKikimrReplication::TReplicationConfig&& config) - : Impl(std::make_shared<TImpl>(id, pathId, std::move(config))) -{ -} - -static auto ParseConfig(const TString& config) { - NKikimrReplication::TReplicationConfig cfg; - Y_VERIFY(cfg.ParseFromString(config)); - return cfg; -} - -TReplication::TReplication(ui64 id, const TPathId& pathId, const TString& config) - : Impl(std::make_shared<TImpl>(id, pathId, ParseConfig(config))) -{ -} - -ui64 TReplication::AddTarget(ETargetKind kind, const TString& srcPath, const TString& dstPath) { - return Impl->AddTarget(kind, srcPath, dstPath); -} - -TReplication::ITarget* TReplication::AddTarget(ui64 id, ETargetKind kind, const TString& srcPath, const TString& dstPath) { - Impl->AddTarget(id, kind, srcPath, dstPath); - return Impl->FindTarget(id); -} - -const TReplication::ITarget* TReplication::FindTarget(ui64 id) const { - return Impl->FindTarget(id); -} - -TReplication::ITarget* TReplication::FindTarget(ui64 id) { - return Impl->FindTarget(id); -} - -void TReplication::Progress(const TActorContext& ctx) { - Impl->Progress(ctx); -} - -void TReplication::Shutdown(const TActorContext& ctx) { - Impl->Shutdown(ctx); -} - -ui64 TReplication::GetId() const { - return Impl->ReplicationId; -} - -void TReplication::SetState(EState state, TString issue) { - Impl->SetState(state, issue); -} - -TReplication::EState TReplication::GetState() const { - return Impl->State; -} - -const TString& TReplication::GetIssue() const { - return Impl->Issue; -} - -void TReplication::SetNextTargetId(ui64 value) { - Impl->NextTargetId = value; -} - -ui64 TReplication::GetNextTargetId() const { - return Impl->NextTargetId; -} - -} // NController -} // NReplication -} // NKikimr - -Y_DECLARE_OUT_SPEC(, NKikimrReplication::TReplicationConfig::TargetCase, stream, value) { - stream << static_cast<int>(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrReplication::TReplicationConfig::CredentialsCase, stream, value) { - stream << static_cast<int>(value); -} +#include "discoverer.h" +#include "replication.h" +#include "target_table.h" +#include "util.h" + +#include <ydb/core/protos/replication.pb.h> +#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/core/util/yverify_stream.h> + +#include <library/cpp/actors/core/events.h> + +#include <util/generic/hash.h> +#include <util/generic/ptr.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TReplication::TImpl { + friend class TReplication; + + template <typename... Args> + ITarget* CreateTarget(ui64 id, ETargetKind kind, Args&&... args) const { + switch (kind) { + case ETargetKind::Table: + return new TTableTarget(ReplicationId, id, std::forward<Args>(args)...); + } + } + + void DiscoverTargets(const TActorContext& ctx) { + switch (Config.GetTargetCase()) { + case NKikimrReplication::TReplicationConfig::kEverything: + return ErrorState("Not implemented"); + + case NKikimrReplication::TReplicationConfig::kSpecific: { + TVector<std::pair<TString, TString>> paths; + for (const auto& target : Config.GetSpecific().GetTargets()) { + paths.emplace_back(target.GetSrcPath(), target.GetDstPath()); + } + + Discoverer = ctx.Register(CreateDiscoverer(ctx.SelfID, ReplicationId, YdbProxy, std::move(paths))); + break; + } + + default: + return ErrorState(TStringBuilder() << "Unexpected targets: " << Config.GetTargetCase()); + } + } + + void ProgressTargets(const TActorContext& ctx) { + for (auto& [_, target] : Targets) { + target->Progress(PathId.OwnerId, YdbProxy, ctx); + } + } + +public: + template <typename T> + explicit TImpl(ui64 id, const TPathId& pathId, T&& config) + : ReplicationId(id) + , PathId(pathId) + , Config(std::forward<T>(config)) + { + } + + template <typename... Args> + ui64 AddTarget(ui64 id, ETargetKind kind, Args&&... args) { + const auto res = Targets.emplace(id, CreateTarget(id, kind, std::forward<Args>(args)...)); + Y_VERIFY_S(res.second, "Duplicate target: " << id); + return id; + } + + template <typename... Args> + ui64 AddTarget(ETargetKind kind, Args&&... args) { + return AddTarget(NextTargetId++, kind, std::forward<Args>(args)...); + } + + ITarget* FindTarget(ui64 id) { + auto it = Targets.find(id); + return it != Targets.end() + ? it->second.Get() + : nullptr; + } + + void Progress(const TActorContext& ctx) { + if (!YdbProxy) { + THolder<IActor> ydbProxy; + switch (Config.GetCredentialsCase()) { + case NKikimrReplication::TReplicationConfig::kStaticCredentials: + ydbProxy.Reset(CreateYdbProxy(Config.GetSrcEndpoint(), Config.GetSrcDatabase(), Config.GetStaticCredentials())); + break; + default: + ErrorState(TStringBuilder() << "Unexpected credentials: " << Config.GetCredentialsCase()); + break; + } + + if (ydbProxy) { + YdbProxy = ctx.Register(ydbProxy.Release()); + } + } + + switch (State) { + case EState::Ready: + if (!Targets) { + return DiscoverTargets(ctx); + } else { + return ProgressTargets(ctx); + } + case EState::Removing: + return; // TODO + case EState::Error: + return; + } + } + + void Shutdown(const TActorContext& ctx) { + for (auto& [_, target] : Targets) { + target->Shutdown(ctx); + } + + for (auto& x : TVector<TActorId>{Discoverer, YdbProxy}) { + if (auto actorId = std::exchange(x, {})) { + ctx.Send(actorId, new TEvents::TEvPoison()); + } + } + } + + void SetState(EState state, TString issue = {}) { + State = state; + Issue = TruncatedIssue(issue); + } + + void ErrorState(TString issue) { + SetState(EState::Error, issue); + } + +private: + const ui64 ReplicationId; + const TPathId PathId; + + NKikimrReplication::TReplicationConfig Config; + EState State = EState::Ready; + TString Issue; + ui64 NextTargetId = 1; + THashMap<ui64, THolder<ITarget>> Targets; + TActorId YdbProxy; + TActorId Discoverer; + +}; // TImpl + +TReplication::TReplication(ui64 id, const TPathId& pathId, const NKikimrReplication::TReplicationConfig& config) + : Impl(std::make_shared<TImpl>(id, pathId, config)) +{ +} + +TReplication::TReplication(ui64 id, const TPathId& pathId, NKikimrReplication::TReplicationConfig&& config) + : Impl(std::make_shared<TImpl>(id, pathId, std::move(config))) +{ +} + +static auto ParseConfig(const TString& config) { + NKikimrReplication::TReplicationConfig cfg; + Y_VERIFY(cfg.ParseFromString(config)); + return cfg; +} + +TReplication::TReplication(ui64 id, const TPathId& pathId, const TString& config) + : Impl(std::make_shared<TImpl>(id, pathId, ParseConfig(config))) +{ +} + +ui64 TReplication::AddTarget(ETargetKind kind, const TString& srcPath, const TString& dstPath) { + return Impl->AddTarget(kind, srcPath, dstPath); +} + +TReplication::ITarget* TReplication::AddTarget(ui64 id, ETargetKind kind, const TString& srcPath, const TString& dstPath) { + Impl->AddTarget(id, kind, srcPath, dstPath); + return Impl->FindTarget(id); +} + +const TReplication::ITarget* TReplication::FindTarget(ui64 id) const { + return Impl->FindTarget(id); +} + +TReplication::ITarget* TReplication::FindTarget(ui64 id) { + return Impl->FindTarget(id); +} + +void TReplication::Progress(const TActorContext& ctx) { + Impl->Progress(ctx); +} + +void TReplication::Shutdown(const TActorContext& ctx) { + Impl->Shutdown(ctx); +} + +ui64 TReplication::GetId() const { + return Impl->ReplicationId; +} + +void TReplication::SetState(EState state, TString issue) { + Impl->SetState(state, issue); +} + +TReplication::EState TReplication::GetState() const { + return Impl->State; +} + +const TString& TReplication::GetIssue() const { + return Impl->Issue; +} + +void TReplication::SetNextTargetId(ui64 value) { + Impl->NextTargetId = value; +} + +ui64 TReplication::GetNextTargetId() const { + return Impl->NextTargetId; +} + +} // NController +} // NReplication +} // NKikimr + +Y_DECLARE_OUT_SPEC(, NKikimrReplication::TReplicationConfig::TargetCase, stream, value) { + stream << static_cast<int>(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrReplication::TReplicationConfig::CredentialsCase, stream, value) { + stream << static_cast<int>(value); +} diff --git a/ydb/core/tx/replication/controller/replication.h b/ydb/core/tx/replication/controller/replication.h index ce4a74e381a..9bc2f529d5d 100644 --- a/ydb/core/tx/replication/controller/replication.h +++ b/ydb/core/tx/replication/controller/replication.h @@ -1,104 +1,104 @@ -#pragma once - -#include "sys_params.h" - -#include <ydb/core/base/defs.h> -#include <ydb/core/base/pathid.h> - -#include <util/generic/ptr.h> - -#include <memory> - -namespace NKikimrReplication { - class TReplicationConfig; -} - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TReplication: public TSimpleRefCount<TReplication> { -public: - using TPtr = TIntrusivePtr<TReplication>; - - enum class EState: ui8 { - Ready, - Removing, - Error = 255 - }; - - enum class ETargetKind: ui8 { - Table, - }; - - enum class EDstState: ui8 { - Creating, - Syncing, - Ready, - Removing, - Error = 255 - }; - - enum class EStreamState: ui8 { - Creating, - Ready, - Removing, - Error = 255 - }; - - class ITarget { - public: - virtual ~ITarget() = default; - - virtual const TString& GetSrcPath() const = 0; - virtual const TString& GetDstPath() const = 0; - - virtual EDstState GetDstState() const = 0; - virtual void SetDstState(const EDstState value) = 0; - - virtual const TPathId& GetDstPathId() const = 0; - virtual void SetDstPathId(const TPathId& value) = 0; - - virtual const TString& GetStreamName() const = 0; - virtual void SetStreamName(const TString& value) = 0; - - virtual EStreamState GetStreamState() const = 0; - virtual void SetStreamState(EStreamState value) = 0; - - virtual const TString& GetIssue() const = 0; - virtual void SetIssue(const TString& value) = 0; - - virtual void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) = 0; - virtual void Shutdown(const TActorContext& ctx) = 0; - }; - -public: - explicit TReplication(ui64 id, const TPathId& pathId, const NKikimrReplication::TReplicationConfig& config); - explicit TReplication(ui64 id, const TPathId& pathId, NKikimrReplication::TReplicationConfig&& config); - explicit TReplication(ui64 id, const TPathId& pathId, const TString& config); - - ui64 AddTarget(ETargetKind kind, const TString& srcPath, const TString& dstPath); - ITarget* AddTarget(ui64 id, ETargetKind kind, const TString& srcPath, const TString& dstPath); - const ITarget* FindTarget(ui64 id) const; - ITarget* FindTarget(ui64 id); - - void Progress(const TActorContext& ctx); - void Shutdown(const TActorContext& ctx); - - ui64 GetId() const; - void SetState(EState state, TString issue = {}); - EState GetState() const; - const TString& GetIssue() const; - - void SetNextTargetId(ui64 value); - ui64 GetNextTargetId() const; - -private: - class TImpl; - std::shared_ptr<TImpl> Impl; - -}; // TReplication - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include "sys_params.h" + +#include <ydb/core/base/defs.h> +#include <ydb/core/base/pathid.h> + +#include <util/generic/ptr.h> + +#include <memory> + +namespace NKikimrReplication { + class TReplicationConfig; +} + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TReplication: public TSimpleRefCount<TReplication> { +public: + using TPtr = TIntrusivePtr<TReplication>; + + enum class EState: ui8 { + Ready, + Removing, + Error = 255 + }; + + enum class ETargetKind: ui8 { + Table, + }; + + enum class EDstState: ui8 { + Creating, + Syncing, + Ready, + Removing, + Error = 255 + }; + + enum class EStreamState: ui8 { + Creating, + Ready, + Removing, + Error = 255 + }; + + class ITarget { + public: + virtual ~ITarget() = default; + + virtual const TString& GetSrcPath() const = 0; + virtual const TString& GetDstPath() const = 0; + + virtual EDstState GetDstState() const = 0; + virtual void SetDstState(const EDstState value) = 0; + + virtual const TPathId& GetDstPathId() const = 0; + virtual void SetDstPathId(const TPathId& value) = 0; + + virtual const TString& GetStreamName() const = 0; + virtual void SetStreamName(const TString& value) = 0; + + virtual EStreamState GetStreamState() const = 0; + virtual void SetStreamState(EStreamState value) = 0; + + virtual const TString& GetIssue() const = 0; + virtual void SetIssue(const TString& value) = 0; + + virtual void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) = 0; + virtual void Shutdown(const TActorContext& ctx) = 0; + }; + +public: + explicit TReplication(ui64 id, const TPathId& pathId, const NKikimrReplication::TReplicationConfig& config); + explicit TReplication(ui64 id, const TPathId& pathId, NKikimrReplication::TReplicationConfig&& config); + explicit TReplication(ui64 id, const TPathId& pathId, const TString& config); + + ui64 AddTarget(ETargetKind kind, const TString& srcPath, const TString& dstPath); + ITarget* AddTarget(ui64 id, ETargetKind kind, const TString& srcPath, const TString& dstPath); + const ITarget* FindTarget(ui64 id) const; + ITarget* FindTarget(ui64 id); + + void Progress(const TActorContext& ctx); + void Shutdown(const TActorContext& ctx); + + ui64 GetId() const; + void SetState(EState state, TString issue = {}); + EState GetState() const; + const TString& GetIssue() const; + + void SetNextTargetId(ui64 value); + ui64 GetNextTargetId() const; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl; + +}; // TReplication + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/schema.h b/ydb/core/tx/replication/controller/schema.h index 93d477e1b3b..163caadd730 100644 --- a/ydb/core/tx/replication/controller/schema.h +++ b/ydb/core/tx/replication/controller/schema.h @@ -1,77 +1,77 @@ -#pragma once - -#include "replication.h" - +#pragma once + +#include "replication.h" + #include <ydb/core/tablet_flat/flat_cxx_database.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -struct TControllerSchema: NIceDb::Schema { - struct SysParams: Table<1> { - struct Id: Column<1, NScheme::NTypeIds::Uint32> {}; - struct IntValue: Column<2, NScheme::NTypeIds::Uint64> {}; - struct TextValue: Column<3, NScheme::NTypeIds::Utf8> {}; - struct BinaryValue: Column<4, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<Id>; - using TColumns = TableColumns<Id, IntValue, TextValue, BinaryValue>; - }; - - struct Replications: Table<2> { - struct Id: Column<1, NScheme::NTypeIds::Uint64> {}; - struct PathOwnerId: Column<2, NScheme::NTypeIds::Uint64> {}; - struct PathLocalId: Column<3, NScheme::NTypeIds::Uint64> {}; - struct Config: Column<4, NScheme::NTypeIds::String> {}; - struct State: Column<5, NScheme::NTypeIds::Uint8> { using Type = TReplication::EState; }; - struct Issue: Column<6, NScheme::NTypeIds::Utf8> {}; - struct NextTargetId: Column<7, NScheme::NTypeIds::Uint64> { static constexpr Type Default = 1; }; - - using TKey = TableKey<Id>; - using TColumns = TableColumns<Id, PathOwnerId, PathLocalId, Config, State, Issue, NextTargetId>; - }; - - struct Targets: Table<3> { - struct ReplicationId: Column<1, NScheme::NTypeIds::Uint64> {}; - struct Id: Column<2, NScheme::NTypeIds::Uint64> {}; - struct Kind: Column<3, NScheme::NTypeIds::Uint8> { using Type = TReplication::ETargetKind; }; - struct SrcPath: Column<4, NScheme::NTypeIds::Utf8> {}; - struct DstPath: Column<5, NScheme::NTypeIds::Utf8> {}; - struct DstState: Column<6, NScheme::NTypeIds::Uint8> { using Type = TReplication::EDstState; }; - struct DstPathOwnerId: Column<7, NScheme::NTypeIds::Uint64> { - using Type = TOwnerId; - static constexpr Type Default = InvalidOwnerId; - }; - struct DstPathLocalId: Column<8, NScheme::NTypeIds::Uint64> { - using Type = TLocalPathId; - static constexpr Type Default = InvalidLocalPathId; - }; - struct Issue: Column<9, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<ReplicationId, Id>; - using TColumns = TableColumns<ReplicationId, Id, Kind, SrcPath, DstPath, DstState, DstPathOwnerId, DstPathLocalId, Issue>; - }; - - struct SrcStreams: Table<4> { - struct ReplicationId: Column<1, NScheme::NTypeIds::Uint64> {}; - struct TargetId: Column<2, NScheme::NTypeIds::Uint64> {}; - struct Name: Column<3, NScheme::NTypeIds::Utf8> {}; - struct State: Column<4, NScheme::NTypeIds::Uint8> { using Type = TReplication::EStreamState; }; - - using TKey = TableKey<ReplicationId, TargetId>; - using TColumns = TableColumns<ReplicationId, TargetId, Name, State>; - }; - - using TTables = SchemaTables< - SysParams, - Replications, - Targets, - SrcStreams - >; - -}; // TControllerSchema - -} // NController -} // NReplication -} // NKikimr + +namespace NKikimr { +namespace NReplication { +namespace NController { + +struct TControllerSchema: NIceDb::Schema { + struct SysParams: Table<1> { + struct Id: Column<1, NScheme::NTypeIds::Uint32> {}; + struct IntValue: Column<2, NScheme::NTypeIds::Uint64> {}; + struct TextValue: Column<3, NScheme::NTypeIds::Utf8> {}; + struct BinaryValue: Column<4, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<Id>; + using TColumns = TableColumns<Id, IntValue, TextValue, BinaryValue>; + }; + + struct Replications: Table<2> { + struct Id: Column<1, NScheme::NTypeIds::Uint64> {}; + struct PathOwnerId: Column<2, NScheme::NTypeIds::Uint64> {}; + struct PathLocalId: Column<3, NScheme::NTypeIds::Uint64> {}; + struct Config: Column<4, NScheme::NTypeIds::String> {}; + struct State: Column<5, NScheme::NTypeIds::Uint8> { using Type = TReplication::EState; }; + struct Issue: Column<6, NScheme::NTypeIds::Utf8> {}; + struct NextTargetId: Column<7, NScheme::NTypeIds::Uint64> { static constexpr Type Default = 1; }; + + using TKey = TableKey<Id>; + using TColumns = TableColumns<Id, PathOwnerId, PathLocalId, Config, State, Issue, NextTargetId>; + }; + + struct Targets: Table<3> { + struct ReplicationId: Column<1, NScheme::NTypeIds::Uint64> {}; + struct Id: Column<2, NScheme::NTypeIds::Uint64> {}; + struct Kind: Column<3, NScheme::NTypeIds::Uint8> { using Type = TReplication::ETargetKind; }; + struct SrcPath: Column<4, NScheme::NTypeIds::Utf8> {}; + struct DstPath: Column<5, NScheme::NTypeIds::Utf8> {}; + struct DstState: Column<6, NScheme::NTypeIds::Uint8> { using Type = TReplication::EDstState; }; + struct DstPathOwnerId: Column<7, NScheme::NTypeIds::Uint64> { + using Type = TOwnerId; + static constexpr Type Default = InvalidOwnerId; + }; + struct DstPathLocalId: Column<8, NScheme::NTypeIds::Uint64> { + using Type = TLocalPathId; + static constexpr Type Default = InvalidLocalPathId; + }; + struct Issue: Column<9, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<ReplicationId, Id>; + using TColumns = TableColumns<ReplicationId, Id, Kind, SrcPath, DstPath, DstState, DstPathOwnerId, DstPathLocalId, Issue>; + }; + + struct SrcStreams: Table<4> { + struct ReplicationId: Column<1, NScheme::NTypeIds::Uint64> {}; + struct TargetId: Column<2, NScheme::NTypeIds::Uint64> {}; + struct Name: Column<3, NScheme::NTypeIds::Utf8> {}; + struct State: Column<4, NScheme::NTypeIds::Uint8> { using Type = TReplication::EStreamState; }; + + using TKey = TableKey<ReplicationId, TargetId>; + using TColumns = TableColumns<ReplicationId, TargetId, Name, State>; + }; + + using TTables = SchemaTables< + SysParams, + Replications, + Targets, + SrcStreams + >; + +}; // TControllerSchema + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/stream_creator.cpp b/ydb/core/tx/replication/controller/stream_creator.cpp index d95c3eb1809..fe03b285f2d 100644 --- a/ydb/core/tx/replication/controller/stream_creator.cpp +++ b/ydb/core/tx/replication/controller/stream_creator.cpp @@ -1,58 +1,58 @@ -#include "stream_creator.h" -#include "private_events.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TStreamCreator: public TActorBootstrapped<TStreamCreator> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::REPLICATION_CONTROLLER_STREAM_CREATOR; - } - - explicit TStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy) - : Parent(parent) - , ReplicationId(rid) - , TargetId(tid) - , YdbProxy(proxy) - { - // TODO: remove it - Y_UNUSED(Parent); - Y_UNUSED(ReplicationId); - Y_UNUSED(TargetId); - Y_UNUSED(YdbProxy); - } - - void Bootstrap() { - // TODO: send request - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - // TODO: handle response - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - const TActorId Parent; - const ui64 ReplicationId; - const ui64 TargetId; - const TActorId YdbProxy; - -}; // TStreamCreator - -IActor* CreateStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy) { - return new TStreamCreator(parent, rid, tid, proxy); -} - -} // NController -} // NReplication -} // NKikimr +#include "stream_creator.h" +#include "private_events.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TStreamCreator: public TActorBootstrapped<TStreamCreator> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::REPLICATION_CONTROLLER_STREAM_CREATOR; + } + + explicit TStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy) + : Parent(parent) + , ReplicationId(rid) + , TargetId(tid) + , YdbProxy(proxy) + { + // TODO: remove it + Y_UNUSED(Parent); + Y_UNUSED(ReplicationId); + Y_UNUSED(TargetId); + Y_UNUSED(YdbProxy); + } + + void Bootstrap() { + // TODO: send request + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + // TODO: handle response + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + const TActorId Parent; + const ui64 ReplicationId; + const ui64 TargetId; + const TActorId YdbProxy; + +}; // TStreamCreator + +IActor* CreateStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy) { + return new TStreamCreator(parent, rid, tid, proxy); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/stream_creator.h b/ydb/core/tx/replication/controller/stream_creator.h index 213d523116f..b08f9da3aa4 100644 --- a/ydb/core/tx/replication/controller/stream_creator.h +++ b/ydb/core/tx/replication/controller/stream_creator.h @@ -1,13 +1,13 @@ -#pragma once - -#include <ydb/core/base/defs.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -IActor* CreateStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy); - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include <ydb/core/base/defs.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +IActor* CreateStreamCreator(const TActorId& parent, ui64 rid, ui64 tid, const TActorId& proxy); + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/sys_params.cpp b/ydb/core/tx/replication/controller/sys_params.cpp index e0abe8e4fdd..2c228e7dbe5 100644 --- a/ydb/core/tx/replication/controller/sys_params.cpp +++ b/ydb/core/tx/replication/controller/sys_params.cpp @@ -1,43 +1,43 @@ -#include "schema.h" -#include "sys_params.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -TSysParams::TSysParams() { - Reset(); -} - -void TSysParams::Reset() { - NextReplicationId = 1; -} - -void TSysParams::Load(ESysParam type, ISysParamLoader* loader) { - switch (type) { - case ESysParam::NextReplicationId: - NextReplicationId = loader->LoadInt(); - break; - default: - break; // ignore - } -} - -void TSysParams::Load(ui32 type, ISysParamLoader* loader) { - Load(ESysParam(type), loader); -} - -ui64 TSysParams::AllocateReplicationId(NIceDb::TNiceDb& db) { - using Schema = TControllerSchema; - const auto result = NextReplicationId++; - - db.Table<Schema::SysParams>().Key(ui32(ESysParam::NextReplicationId)).Update( - NIceDb::TUpdate<Schema::SysParams::IntValue>(NextReplicationId) - ); - - return result; -} - -} // NController -} // NReplication -} // NKikimr +#include "schema.h" +#include "sys_params.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +TSysParams::TSysParams() { + Reset(); +} + +void TSysParams::Reset() { + NextReplicationId = 1; +} + +void TSysParams::Load(ESysParam type, ISysParamLoader* loader) { + switch (type) { + case ESysParam::NextReplicationId: + NextReplicationId = loader->LoadInt(); + break; + default: + break; // ignore + } +} + +void TSysParams::Load(ui32 type, ISysParamLoader* loader) { + Load(ESysParam(type), loader); +} + +ui64 TSysParams::AllocateReplicationId(NIceDb::TNiceDb& db) { + using Schema = TControllerSchema; + const auto result = NextReplicationId++; + + db.Table<Schema::SysParams>().Key(ui32(ESysParam::NextReplicationId)).Update( + NIceDb::TUpdate<Schema::SysParams::IntValue>(NextReplicationId) + ); + + return result; +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/sys_params.h b/ydb/core/tx/replication/controller/sys_params.h index 4eaf6f7bdf8..f8226fbbf3b 100644 --- a/ydb/core/tx/replication/controller/sys_params.h +++ b/ydb/core/tx/replication/controller/sys_params.h @@ -1,42 +1,42 @@ -#pragma once - -#include <util/generic/fwd.h> - -namespace NKikimr { - -namespace NIceDb { - class TNiceDb; -} - -namespace NReplication { -namespace NController { - -class ISysParamLoader { -public: - virtual ~ISysParamLoader() = default; - virtual ui64 LoadInt() = 0; - virtual TString LoadText() = 0; - virtual TString LoadBinary() = 0; -}; - -class TSysParams { - enum class ESysParam: ui32 { - NextReplicationId = 1, - }; - -public: - TSysParams(); - void Reset(); - void Load(ESysParam type, ISysParamLoader* loader); - void Load(ui32 type, ISysParamLoader* loader); - - ui64 AllocateReplicationId(NIceDb::TNiceDb& db); - -private: - ui64 NextReplicationId; - -}; // TSysParams - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include <util/generic/fwd.h> + +namespace NKikimr { + +namespace NIceDb { + class TNiceDb; +} + +namespace NReplication { +namespace NController { + +class ISysParamLoader { +public: + virtual ~ISysParamLoader() = default; + virtual ui64 LoadInt() = 0; + virtual TString LoadText() = 0; + virtual TString LoadBinary() = 0; +}; + +class TSysParams { + enum class ESysParam: ui32 { + NextReplicationId = 1, + }; + +public: + TSysParams(); + void Reset(); + void Load(ESysParam type, ISysParamLoader* loader); + void Load(ui32 type, ISysParamLoader* loader); + + ui64 AllocateReplicationId(NIceDb::TNiceDb& db); + +private: + ui64 NextReplicationId; + +}; // TSysParams + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_base.cpp b/ydb/core/tx/replication/controller/target_base.cpp index 7cd3ba7ccc8..0d3d3ee67bb 100644 --- a/ydb/core/tx/replication/controller/target_base.cpp +++ b/ydb/core/tx/replication/controller/target_base.cpp @@ -1,109 +1,109 @@ -#include "dst_creator.h" -#include "target_base.h" -#include "util.h" - -#include <library/cpp/actors/core/events.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -using EDstState = TReplication::EDstState; -using EStreamState = TReplication::EStreamState; - -TTargetBase::TTargetBase(ETargetKind kind, ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath) - : Kind(kind) - , ReplicationId(rid) - , TargetId(tid) - , SrcPath(srcPath) - , DstPath(dstPath) -{ -} - -const TString& TTargetBase::GetSrcPath() const { - return SrcPath; -} - -const TString& TTargetBase::GetDstPath() const { - return DstPath; -} - -EDstState TTargetBase::GetDstState() const { - return DstState; -} - -void TTargetBase::SetDstState(const EDstState value) { - DstState = value; -} - -const TPathId& TTargetBase::GetDstPathId() const { - return DstPathId; -} - -void TTargetBase::SetDstPathId(const TPathId& value) { - DstPathId = value; -} - -const TString& TTargetBase::GetStreamName() const { - return StreamName; -} - -void TTargetBase::SetStreamName(const TString& value) { - StreamName = value; -} - -EStreamState TTargetBase::GetStreamState() const { - return StreamState; -} - -void TTargetBase::SetStreamState(EStreamState value) { - StreamState = value; -} - -const TString& TTargetBase::GetIssue() const { - return Issue; -} - -void TTargetBase::SetIssue(const TString& value) { - Issue = value; - TruncatedIssue(Issue); -} - -ui64 TTargetBase::GetReplicationId() const { - return ReplicationId; -} - -ui64 TTargetBase::GetTargetId() const { - return TargetId; -} - -void TTargetBase::Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) { - switch (DstState) { - case EDstState::Creating: - if (!DstCreator) { - DstCreator = ctx.Register(CreateDstCreator(ctx.SelfID, schemeShardId, proxy, - ReplicationId, TargetId, Kind, SrcPath, DstPath)); - } - break; - case EDstState::Syncing: - break; // TODO - case EDstState::Ready: - break; // TODO - case EDstState::Removing: - break; // TODO - case EDstState::Error: - break; - } -} - -void TTargetBase::Shutdown(const TActorContext& ctx) { - for (auto& x : TVector<TActorId>{DstCreator, DstRemover}) { - if (auto actorId = std::exchange(x, {})) { - ctx.Send(actorId, new TEvents::TEvPoison()); - } - } -} - -} // NController -} // NReplication -} // NKikimr +#include "dst_creator.h" +#include "target_base.h" +#include "util.h" + +#include <library/cpp/actors/core/events.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +using EDstState = TReplication::EDstState; +using EStreamState = TReplication::EStreamState; + +TTargetBase::TTargetBase(ETargetKind kind, ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath) + : Kind(kind) + , ReplicationId(rid) + , TargetId(tid) + , SrcPath(srcPath) + , DstPath(dstPath) +{ +} + +const TString& TTargetBase::GetSrcPath() const { + return SrcPath; +} + +const TString& TTargetBase::GetDstPath() const { + return DstPath; +} + +EDstState TTargetBase::GetDstState() const { + return DstState; +} + +void TTargetBase::SetDstState(const EDstState value) { + DstState = value; +} + +const TPathId& TTargetBase::GetDstPathId() const { + return DstPathId; +} + +void TTargetBase::SetDstPathId(const TPathId& value) { + DstPathId = value; +} + +const TString& TTargetBase::GetStreamName() const { + return StreamName; +} + +void TTargetBase::SetStreamName(const TString& value) { + StreamName = value; +} + +EStreamState TTargetBase::GetStreamState() const { + return StreamState; +} + +void TTargetBase::SetStreamState(EStreamState value) { + StreamState = value; +} + +const TString& TTargetBase::GetIssue() const { + return Issue; +} + +void TTargetBase::SetIssue(const TString& value) { + Issue = value; + TruncatedIssue(Issue); +} + +ui64 TTargetBase::GetReplicationId() const { + return ReplicationId; +} + +ui64 TTargetBase::GetTargetId() const { + return TargetId; +} + +void TTargetBase::Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) { + switch (DstState) { + case EDstState::Creating: + if (!DstCreator) { + DstCreator = ctx.Register(CreateDstCreator(ctx.SelfID, schemeShardId, proxy, + ReplicationId, TargetId, Kind, SrcPath, DstPath)); + } + break; + case EDstState::Syncing: + break; // TODO + case EDstState::Ready: + break; // TODO + case EDstState::Removing: + break; // TODO + case EDstState::Error: + break; + } +} + +void TTargetBase::Shutdown(const TActorContext& ctx) { + for (auto& x : TVector<TActorId>{DstCreator, DstRemover}) { + if (auto actorId = std::exchange(x, {})) { + ctx.Send(actorId, new TEvents::TEvPoison()); + } + } +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_base.h b/ydb/core/tx/replication/controller/target_base.h index afc811deb67..c6fa0ff0859 100644 --- a/ydb/core/tx/replication/controller/target_base.h +++ b/ydb/core/tx/replication/controller/target_base.h @@ -1,63 +1,63 @@ -#pragma once - -#include "replication.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TTargetBase: public TReplication::ITarget { -protected: - using ETargetKind = TReplication::ETargetKind; - using EDstState = TReplication::EDstState; - using EStreamState = TReplication::EStreamState; - -public: - explicit TTargetBase(ETargetKind kind, ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath); - - const TString& GetSrcPath() const override; - const TString& GetDstPath() const override; - - EDstState GetDstState() const override; - void SetDstState(const EDstState value) override; - - const TPathId& GetDstPathId() const override; - void SetDstPathId(const TPathId& value) override; - - const TString& GetStreamName() const override; - void SetStreamName(const TString& value) override; - - EStreamState GetStreamState() const override; - void SetStreamState(EStreamState value) override; - - const TString& GetIssue() const override; - void SetIssue(const TString& value) override; - - void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) override; - void Shutdown(const TActorContext& ctx) override; - -protected: - ui64 GetReplicationId() const; - ui64 GetTargetId() const; - -private: - const ETargetKind Kind; - const ui64 ReplicationId; - const ui64 TargetId; - const TString SrcPath; - const TString DstPath; - - EDstState DstState = EDstState::Creating; - TPathId DstPathId; - TString StreamName; - EStreamState StreamState = EStreamState::Ready; - TString Issue; - - TActorId DstCreator; - TActorId DstRemover; - -}; // TTargetBase - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include "replication.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TTargetBase: public TReplication::ITarget { +protected: + using ETargetKind = TReplication::ETargetKind; + using EDstState = TReplication::EDstState; + using EStreamState = TReplication::EStreamState; + +public: + explicit TTargetBase(ETargetKind kind, ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath); + + const TString& GetSrcPath() const override; + const TString& GetDstPath() const override; + + EDstState GetDstState() const override; + void SetDstState(const EDstState value) override; + + const TPathId& GetDstPathId() const override; + void SetDstPathId(const TPathId& value) override; + + const TString& GetStreamName() const override; + void SetStreamName(const TString& value) override; + + EStreamState GetStreamState() const override; + void SetStreamState(EStreamState value) override; + + const TString& GetIssue() const override; + void SetIssue(const TString& value) override; + + void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) override; + void Shutdown(const TActorContext& ctx) override; + +protected: + ui64 GetReplicationId() const; + ui64 GetTargetId() const; + +private: + const ETargetKind Kind; + const ui64 ReplicationId; + const ui64 TargetId; + const TString SrcPath; + const TString DstPath; + + EDstState DstState = EDstState::Creating; + TPathId DstPathId; + TString StreamName; + EStreamState StreamState = EStreamState::Ready; + TString Issue; + + TActorId DstCreator; + TActorId DstRemover; + +}; // TTargetBase + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_table.cpp b/ydb/core/tx/replication/controller/target_table.cpp index ade125592c0..a8613ac7ac2 100644 --- a/ydb/core/tx/replication/controller/target_table.cpp +++ b/ydb/core/tx/replication/controller/target_table.cpp @@ -1,14 +1,14 @@ -#include "target_table.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -TTableTarget::TTableTarget(ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath) - : TTargetWithStream(ETargetKind::Table, rid, tid, srcPath, dstPath) -{ -} - -} // NController -} // NReplication -} // NKikimr +#include "target_table.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +TTableTarget::TTableTarget(ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath) + : TTargetWithStream(ETargetKind::Table, rid, tid, srcPath, dstPath) +{ +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_table.h b/ydb/core/tx/replication/controller/target_table.h index 7afaa2f4d03..85928bcefc8 100644 --- a/ydb/core/tx/replication/controller/target_table.h +++ b/ydb/core/tx/replication/controller/target_table.h @@ -1,17 +1,17 @@ -#pragma once - -#include "target_with_stream.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TTableTarget: public TTargetWithStream { -public: - explicit TTableTarget(ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath); - -}; // TTableTarget - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include "target_with_stream.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TTableTarget: public TTargetWithStream { +public: + explicit TTableTarget(ui64 rid, ui64 tid, const TString& srcPath, const TString& dstPath); + +}; // TTableTarget + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_with_stream.cpp b/ydb/core/tx/replication/controller/target_with_stream.cpp index 7a7cf783370..72d4fe438a8 100644 --- a/ydb/core/tx/replication/controller/target_with_stream.cpp +++ b/ydb/core/tx/replication/controller/target_with_stream.cpp @@ -1,43 +1,43 @@ -#include "private_events.h" -#include "stream_creator.h" -#include "target_with_stream.h" - -#include <library/cpp/actors/core/events.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -void TTargetWithStream::Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) { - switch (GetStreamState()) { - case EStreamState::Creating: - if (GetStreamName().empty() && !NameAssignmentInProcess) { - ctx.Send(ctx.SelfID, new TEvPrivate::TEvAssignStreamName(GetReplicationId(), GetTargetId())); - NameAssignmentInProcess = true; - } else if (!StreamCreator) { - StreamCreator = ctx.Register(CreateStreamCreator(ctx.SelfID, GetReplicationId(), GetTargetId(), proxy)); - } - return; - case EStreamState::Removing: - return; // TODO - case EStreamState::Ready: - case EStreamState::Error: - break; - } - - TTargetBase::Progress(schemeShardId, proxy, ctx); -} - -void TTargetWithStream::Shutdown(const TActorContext& ctx) { - for (auto& x : TVector<TActorId>{StreamCreator, StreamRemover}) { - if (auto actorId = std::exchange(x, {})) { - ctx.Send(actorId, new TEvents::TEvPoison()); - } - } - - TTargetBase::Shutdown(ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "private_events.h" +#include "stream_creator.h" +#include "target_with_stream.h" + +#include <library/cpp/actors/core/events.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +void TTargetWithStream::Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) { + switch (GetStreamState()) { + case EStreamState::Creating: + if (GetStreamName().empty() && !NameAssignmentInProcess) { + ctx.Send(ctx.SelfID, new TEvPrivate::TEvAssignStreamName(GetReplicationId(), GetTargetId())); + NameAssignmentInProcess = true; + } else if (!StreamCreator) { + StreamCreator = ctx.Register(CreateStreamCreator(ctx.SelfID, GetReplicationId(), GetTargetId(), proxy)); + } + return; + case EStreamState::Removing: + return; // TODO + case EStreamState::Ready: + case EStreamState::Error: + break; + } + + TTargetBase::Progress(schemeShardId, proxy, ctx); +} + +void TTargetWithStream::Shutdown(const TActorContext& ctx) { + for (auto& x : TVector<TActorId>{StreamCreator, StreamRemover}) { + if (auto actorId = std::exchange(x, {})) { + ctx.Send(actorId, new TEvents::TEvPoison()); + } + } + + TTargetBase::Shutdown(ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/target_with_stream.h b/ydb/core/tx/replication/controller/target_with_stream.h index d129706cd79..df58baddfb2 100644 --- a/ydb/core/tx/replication/controller/target_with_stream.h +++ b/ydb/core/tx/replication/controller/target_with_stream.h @@ -1,30 +1,30 @@ -#pragma once - -#include "target_base.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TTargetWithStream: public TTargetBase { -public: - template <typename... Args> - explicit TTargetWithStream(Args&&... args) - : TTargetBase(std::forward<Args>(args)...) - { - SetStreamState(EStreamState::Creating); - } - - void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) override; - void Shutdown(const TActorContext& ctx) override; - -private: - bool NameAssignmentInProcess = false; - TActorId StreamCreator; - TActorId StreamRemover; - -}; // TTargetWithStream - -} // NController -} // NReplication -} // NKikimr +#pragma once + +#include "target_base.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TTargetWithStream: public TTargetBase { +public: + template <typename... Args> + explicit TTargetWithStream(Args&&... args) + : TTargetBase(std::forward<Args>(args)...) + { + SetStreamState(EStreamState::Creating); + } + + void Progress(ui64 schemeShardId, const TActorId& proxy, const TActorContext& ctx) override; + void Shutdown(const TActorContext& ctx) override; + +private: + bool NameAssignmentInProcess = false; + TActorId StreamCreator; + TActorId StreamRemover; + +}; // TTargetWithStream + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_assign_stream_name.cpp b/ydb/core/tx/replication/controller/tx_assign_stream_name.cpp index 794a8ca8318..0062f82b456 100644 --- a/ydb/core/tx/replication/controller/tx_assign_stream_name.cpp +++ b/ydb/core/tx/replication/controller/tx_assign_stream_name.cpp @@ -1,83 +1,83 @@ -#include "controller_impl.h" - -#include <util/generic/guid.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxAssignStreamName: public TTxBase { - TEvPrivate::TEvAssignStreamName::TPtr Ev; - TReplication::TPtr Replication; - -public: - explicit TTxAssignStreamName(TController* self, TEvPrivate::TEvAssignStreamName::TPtr& ev) - : TTxBase("TxAssignStreamName", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_ASSIGN_STREAM_NAME; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto rid = Ev->Get()->ReplicationId; - const auto tid = Ev->Get()->TargetId; - - Replication = Self->Find(rid); - if (!Replication) { - CLOG_W(ctx, "Unknown replication" - << ": rid# " << rid); - return true; - } - - auto* target = Replication->FindTarget(tid); - if (!target) { - CLOG_W(ctx, "Unknown target" - << ": rid# " << rid - << ", tid# " << tid); - return true; - } - - if (!target->GetStreamName().empty()) { - CLOG_W(ctx, "Stream name already assigned" - << ": rid# " << rid - << ", tid# " << tid); - return true; - } - - target->SetStreamName(CreateGuidAsString()); - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::SrcStreams>().Key(rid, tid).Update( - NIceDb::TUpdate<Schema::SrcStreams::Name>(target->GetStreamName()) - ); - - CLOG_N(ctx, "Stream name assigned" - << ": rid# " << rid - << ", tid# " << tid - << ", name# " << target->GetStreamName()); - - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxAssignStreamName - -void TController::RunTxAssignStreamName(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxAssignStreamName(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +#include <util/generic/guid.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxAssignStreamName: public TTxBase { + TEvPrivate::TEvAssignStreamName::TPtr Ev; + TReplication::TPtr Replication; + +public: + explicit TTxAssignStreamName(TController* self, TEvPrivate::TEvAssignStreamName::TPtr& ev) + : TTxBase("TxAssignStreamName", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_ASSIGN_STREAM_NAME; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto rid = Ev->Get()->ReplicationId; + const auto tid = Ev->Get()->TargetId; + + Replication = Self->Find(rid); + if (!Replication) { + CLOG_W(ctx, "Unknown replication" + << ": rid# " << rid); + return true; + } + + auto* target = Replication->FindTarget(tid); + if (!target) { + CLOG_W(ctx, "Unknown target" + << ": rid# " << rid + << ", tid# " << tid); + return true; + } + + if (!target->GetStreamName().empty()) { + CLOG_W(ctx, "Stream name already assigned" + << ": rid# " << rid + << ", tid# " << tid); + return true; + } + + target->SetStreamName(CreateGuidAsString()); + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::SrcStreams>().Key(rid, tid).Update( + NIceDb::TUpdate<Schema::SrcStreams::Name>(target->GetStreamName()) + ); + + CLOG_N(ctx, "Stream name assigned" + << ": rid# " << rid + << ", tid# " << tid + << ", name# " << target->GetStreamName()); + + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxAssignStreamName + +void TController::RunTxAssignStreamName(TEvPrivate::TEvAssignStreamName::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxAssignStreamName(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_create_dst_result.cpp b/ydb/core/tx/replication/controller/tx_create_dst_result.cpp index 60d3404ed92..5af19e35b7c 100644 --- a/ydb/core/tx/replication/controller/tx_create_dst_result.cpp +++ b/ydb/core/tx/replication/controller/tx_create_dst_result.cpp @@ -1,95 +1,95 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxCreateDstResult: public TTxBase { - TEvPrivate::TEvCreateDstResult::TPtr Ev; - TReplication::TPtr Replication; - -public: - explicit TTxCreateDstResult(TController* self, TEvPrivate::TEvCreateDstResult::TPtr& ev) - : TTxBase("TxCreateDstResult", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CREATE_DST_RESULT; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto rid = Ev->Get()->ReplicationId; - const auto tid = Ev->Get()->TargetId; - - Replication = Self->Find(rid); - if (!Replication) { - CLOG_W(ctx, "Unknown replication" - << ": rid# " << rid); - return true; - } - - auto* target = Replication->FindTarget(tid); - if (!target) { - CLOG_W(ctx, "Unknown target" - << ": rid# " << rid - << ", tid# " << tid); - return true; - } - - if (Ev->Get()->IsSuccess()) { - target->SetDstPathId(Ev->Get()->DstPathId); - target->SetDstState(TReplication::EDstState::Ready); - - CLOG_N(ctx, "Target dst created" - << ": rid# " << rid - << ", tid# " << tid - << ", pathId# " << Ev->Get()->DstPathId); - } else { - Replication->SetState(TReplication::EState::Error); - target->SetDstState(TReplication::EDstState::Error); - target->SetIssue(TStringBuilder() << "Create dst error" - << ": " << NKikimrScheme::EStatus_Name(Ev->Get()->Status) - << ", " << Ev->Get()->Error); - - CLOG_E(ctx, "Create dst error" - << ": rid# " << rid - << ", tid# " << tid - << ", " << NKikimrScheme::EStatus_Name(Ev->Get()->Status) - << ", " << Ev->Get()->Error); - } - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Replications>().Key(rid).Update( - NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()) - ); - db.Table<Schema::Targets>().Key(rid, tid).Update( - NIceDb::TUpdate<Schema::Targets::DstPathOwnerId>(target->GetDstPathId().OwnerId), - NIceDb::TUpdate<Schema::Targets::DstPathLocalId>(target->GetDstPathId().LocalPathId), - NIceDb::TUpdate<Schema::Targets::DstState>(target->GetDstState()), - NIceDb::TUpdate<Schema::Targets::Issue>(target->GetIssue()) - ); - - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxCreateDstResult - -void TController::RunTxCreateDstResult(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxCreateDstResult(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxCreateDstResult: public TTxBase { + TEvPrivate::TEvCreateDstResult::TPtr Ev; + TReplication::TPtr Replication; + +public: + explicit TTxCreateDstResult(TController* self, TEvPrivate::TEvCreateDstResult::TPtr& ev) + : TTxBase("TxCreateDstResult", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CREATE_DST_RESULT; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto rid = Ev->Get()->ReplicationId; + const auto tid = Ev->Get()->TargetId; + + Replication = Self->Find(rid); + if (!Replication) { + CLOG_W(ctx, "Unknown replication" + << ": rid# " << rid); + return true; + } + + auto* target = Replication->FindTarget(tid); + if (!target) { + CLOG_W(ctx, "Unknown target" + << ": rid# " << rid + << ", tid# " << tid); + return true; + } + + if (Ev->Get()->IsSuccess()) { + target->SetDstPathId(Ev->Get()->DstPathId); + target->SetDstState(TReplication::EDstState::Ready); + + CLOG_N(ctx, "Target dst created" + << ": rid# " << rid + << ", tid# " << tid + << ", pathId# " << Ev->Get()->DstPathId); + } else { + Replication->SetState(TReplication::EState::Error); + target->SetDstState(TReplication::EDstState::Error); + target->SetIssue(TStringBuilder() << "Create dst error" + << ": " << NKikimrScheme::EStatus_Name(Ev->Get()->Status) + << ", " << Ev->Get()->Error); + + CLOG_E(ctx, "Create dst error" + << ": rid# " << rid + << ", tid# " << tid + << ", " << NKikimrScheme::EStatus_Name(Ev->Get()->Status) + << ", " << Ev->Get()->Error); + } + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::Replications>().Key(rid).Update( + NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()) + ); + db.Table<Schema::Targets>().Key(rid, tid).Update( + NIceDb::TUpdate<Schema::Targets::DstPathOwnerId>(target->GetDstPathId().OwnerId), + NIceDb::TUpdate<Schema::Targets::DstPathLocalId>(target->GetDstPathId().LocalPathId), + NIceDb::TUpdate<Schema::Targets::DstState>(target->GetDstState()), + NIceDb::TUpdate<Schema::Targets::Issue>(target->GetIssue()) + ); + + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxCreateDstResult + +void TController::RunTxCreateDstResult(TEvPrivate::TEvCreateDstResult::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxCreateDstResult(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_create_replication.cpp b/ydb/core/tx/replication/controller/tx_create_replication.cpp index fbbe6a9e2db..6a987478a34 100644 --- a/ydb/core/tx/replication/controller/tx_create_replication.cpp +++ b/ydb/core/tx/replication/controller/tx_create_replication.cpp @@ -1,78 +1,78 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxCreateReplication: public TTxBase { - TEvController::TEvCreateReplication::TPtr Ev; - THolder<TEvController::TEvCreateReplicationResult> Result; - TReplication::TPtr Replication; - -public: - explicit TTxCreateReplication(TController* self, TEvController::TEvCreateReplication::TPtr& ev) - : TTxBase("TxCreateReplication", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CREATE_REPLICATION; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto& record = Ev->Get()->Record; - Result = MakeHolder<TEvController::TEvCreateReplicationResult>(); - Result->Record.MutableOperationId()->CopyFrom(record.GetOperationId()); - Result->Record.SetOrigin(Self->TabletID()); - - const auto pathId = PathIdFromPathId(record.GetPathId()); - if (Self->Find(pathId)) { - CLOG_W(ctx, "Replication already exists" - << ": pathId# " << pathId); - - Result->Record.SetStatus(NKikimrReplication::TEvCreateReplicationResult::ALREADY_EXISTS); - return true; - } - - NIceDb::TNiceDb db(txc.DB); - const auto rid = Self->SysParams.AllocateReplicationId(db); - - Replication = Self->Add(rid, pathId, record.GetConfig()); - db.Table<Schema::Replications>().Key(rid).Update( - NIceDb::TUpdate<Schema::Replications::PathOwnerId>(pathId.OwnerId), - NIceDb::TUpdate<Schema::Replications::PathLocalId>(pathId.LocalPathId), - NIceDb::TUpdate<Schema::Replications::Config>(record.GetConfig().SerializeAsString()) - ); - - CLOG_N(ctx, "Add replication" - << ": rid# " << rid - << ", pathId# " << pathId); - - Result->Record.SetStatus(NKikimrReplication::TEvCreateReplicationResult::SUCCESS); - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Result) { - ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); - } - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxCreateReplication - -void TController::RunTxCreateReplication(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxCreateReplication(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxCreateReplication: public TTxBase { + TEvController::TEvCreateReplication::TPtr Ev; + THolder<TEvController::TEvCreateReplicationResult> Result; + TReplication::TPtr Replication; + +public: + explicit TTxCreateReplication(TController* self, TEvController::TEvCreateReplication::TPtr& ev) + : TTxBase("TxCreateReplication", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CREATE_REPLICATION; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto& record = Ev->Get()->Record; + Result = MakeHolder<TEvController::TEvCreateReplicationResult>(); + Result->Record.MutableOperationId()->CopyFrom(record.GetOperationId()); + Result->Record.SetOrigin(Self->TabletID()); + + const auto pathId = PathIdFromPathId(record.GetPathId()); + if (Self->Find(pathId)) { + CLOG_W(ctx, "Replication already exists" + << ": pathId# " << pathId); + + Result->Record.SetStatus(NKikimrReplication::TEvCreateReplicationResult::ALREADY_EXISTS); + return true; + } + + NIceDb::TNiceDb db(txc.DB); + const auto rid = Self->SysParams.AllocateReplicationId(db); + + Replication = Self->Add(rid, pathId, record.GetConfig()); + db.Table<Schema::Replications>().Key(rid).Update( + NIceDb::TUpdate<Schema::Replications::PathOwnerId>(pathId.OwnerId), + NIceDb::TUpdate<Schema::Replications::PathLocalId>(pathId.LocalPathId), + NIceDb::TUpdate<Schema::Replications::Config>(record.GetConfig().SerializeAsString()) + ); + + CLOG_N(ctx, "Add replication" + << ": rid# " << rid + << ", pathId# " << pathId); + + Result->Record.SetStatus(NKikimrReplication::TEvCreateReplicationResult::SUCCESS); + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Result) { + ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); + } + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxCreateReplication + +void TController::RunTxCreateReplication(TEvController::TEvCreateReplication::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxCreateReplication(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_create_stream_result.cpp b/ydb/core/tx/replication/controller/tx_create_stream_result.cpp index d22c844c7fe..41fbdab6b9a 100644 --- a/ydb/core/tx/replication/controller/tx_create_stream_result.cpp +++ b/ydb/core/tx/replication/controller/tx_create_stream_result.cpp @@ -1,89 +1,89 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxCreateStreamResult: public TTxBase { - TEvPrivate::TEvCreateStreamResult::TPtr Ev; - TReplication::TPtr Replication; - -public: - explicit TTxCreateStreamResult(TController* self, TEvPrivate::TEvCreateStreamResult::TPtr& ev) - : TTxBase("TxCreateStreamResult", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CREATE_STREAM_RESULT; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto rid = Ev->Get()->ReplicationId; - const auto tid = Ev->Get()->TargetId; - - Replication = Self->Find(rid); - if (!Replication) { - CLOG_W(ctx, "Unknown replication" - << ": rid# " << rid); - return true; - } - - auto* target = Replication->FindTarget(tid); - if (!target) { - CLOG_W(ctx, "Unknown target" - << ": rid# " << rid - << ", tid# " << tid); - return true; - } - - if (Ev->Get()->IsSuccess()) { - target->SetStreamState(TReplication::EStreamState::Ready); - - CLOG_N(ctx, "Stream created" - << ": rid# " << rid - << ", tid# " << tid); - } else { - const auto& status = Ev->Get()->Status; - - Replication->SetState(TReplication::EState::Error); - target->SetStreamState(TReplication::EStreamState::Error); - target->SetIssue(TStringBuilder() << "Create stream error" - << ": " << status.GetStatus() - << ", " << status.GetIssues().ToOneLineString()); - - CLOG_E(ctx, "Create stream error" - << ": rid# " << rid - << ", tid# " << tid - << ", status# " << status.GetStatus() - << ", issue# " << status.GetIssues().ToOneLineString()); - } - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::SrcStreams>().Key(rid, tid).Update<Schema::SrcStreams::State>(target->GetStreamState()); - db.Table<Schema::Targets>().Key(rid, tid).Update<Schema::Targets::Issue>(target->GetIssue()); - db.Table<Schema::Replications>().Key(rid).Update<Schema::Replications::State>(Replication->GetState()); - - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxCreateStreamResult - -void TController::RunTxCreateStreamResult(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxCreateStreamResult(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxCreateStreamResult: public TTxBase { + TEvPrivate::TEvCreateStreamResult::TPtr Ev; + TReplication::TPtr Replication; + +public: + explicit TTxCreateStreamResult(TController* self, TEvPrivate::TEvCreateStreamResult::TPtr& ev) + : TTxBase("TxCreateStreamResult", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CREATE_STREAM_RESULT; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto rid = Ev->Get()->ReplicationId; + const auto tid = Ev->Get()->TargetId; + + Replication = Self->Find(rid); + if (!Replication) { + CLOG_W(ctx, "Unknown replication" + << ": rid# " << rid); + return true; + } + + auto* target = Replication->FindTarget(tid); + if (!target) { + CLOG_W(ctx, "Unknown target" + << ": rid# " << rid + << ", tid# " << tid); + return true; + } + + if (Ev->Get()->IsSuccess()) { + target->SetStreamState(TReplication::EStreamState::Ready); + + CLOG_N(ctx, "Stream created" + << ": rid# " << rid + << ", tid# " << tid); + } else { + const auto& status = Ev->Get()->Status; + + Replication->SetState(TReplication::EState::Error); + target->SetStreamState(TReplication::EStreamState::Error); + target->SetIssue(TStringBuilder() << "Create stream error" + << ": " << status.GetStatus() + << ", " << status.GetIssues().ToOneLineString()); + + CLOG_E(ctx, "Create stream error" + << ": rid# " << rid + << ", tid# " << tid + << ", status# " << status.GetStatus() + << ", issue# " << status.GetIssues().ToOneLineString()); + } + + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::SrcStreams>().Key(rid, tid).Update<Schema::SrcStreams::State>(target->GetStreamState()); + db.Table<Schema::Targets>().Key(rid, tid).Update<Schema::Targets::Issue>(target->GetIssue()); + db.Table<Schema::Replications>().Key(rid).Update<Schema::Replications::State>(Replication->GetState()); + + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxCreateStreamResult + +void TController::RunTxCreateStreamResult(TEvPrivate::TEvCreateStreamResult::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxCreateStreamResult(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_discovery_result.cpp b/ydb/core/tx/replication/controller/tx_discovery_result.cpp index 7be78056eff..14c63301ef1 100644 --- a/ydb/core/tx/replication/controller/tx_discovery_result.cpp +++ b/ydb/core/tx/replication/controller/tx_discovery_result.cpp @@ -1,93 +1,93 @@ -#include "controller_impl.h" -#include "util.h" - -#include <util/string/join.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxDiscoveryResult: public TTxBase { - TEvPrivate::TEvDiscoveryResult::TPtr Ev; - TReplication::TPtr Replication; - -public: - explicit TTxDiscoveryResult(TController* self, TEvPrivate::TEvDiscoveryResult::TPtr& ev) - : TTxBase("TxDiscovertResult", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_DISCOVERY_RESULT; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto rid = Ev->Get()->ReplicationId; - - Replication = Self->Find(rid); - if (!Replication) { - CLOG_W(ctx, "Unknown replication" - << ": rid# " << rid); - return true; - } - - NIceDb::TNiceDb db(txc.DB); - - if (Ev->Get()->IsSuccess()) { - for (const auto& target : Ev->Get()->ToAdd) { - const auto kind = TargetKindFromEntryType(target.first.Type); - const auto& srcPath = target.first.Name; - const auto& dstPath = target.second; - - const auto tid = Replication->AddTarget(kind, srcPath, dstPath); - db.Table<Schema::Targets>().Key(rid, tid).Update( - NIceDb::TUpdate<Schema::Targets::Kind>(kind), - NIceDb::TUpdate<Schema::Targets::SrcPath>(srcPath), - NIceDb::TUpdate<Schema::Targets::DstPath>(dstPath) - ); - - CLOG_N(ctx, "Add target" - << ": rid# " << rid - << ", tid# " << tid - << ", kind# " << kind - << ", srcPath# " << srcPath - << ", dstPath# " << dstPath); - } - } else { - const auto error = JoinSeq(", ", Ev->Get()->Failed); - Replication->SetState(TReplication::EState::Error, TStringBuilder() << "Discovery error: " << error); - - CLOG_E(ctx, "Discovery error" - << ": rid# " << rid - << ", error# " << error); - } - - db.Table<Schema::Replications>().Key(Replication->GetId()).Update( - NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()), - NIceDb::TUpdate<Schema::Replications::Issue>(Replication->GetIssue()), - NIceDb::TUpdate<Schema::Replications::NextTargetId>(Replication->GetNextTargetId()) - ); - - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxDiscoveryResult - -void TController::RunTxDiscoveryResult(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxDiscoveryResult(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" +#include "util.h" + +#include <util/string/join.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxDiscoveryResult: public TTxBase { + TEvPrivate::TEvDiscoveryResult::TPtr Ev; + TReplication::TPtr Replication; + +public: + explicit TTxDiscoveryResult(TController* self, TEvPrivate::TEvDiscoveryResult::TPtr& ev) + : TTxBase("TxDiscovertResult", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_DISCOVERY_RESULT; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto rid = Ev->Get()->ReplicationId; + + Replication = Self->Find(rid); + if (!Replication) { + CLOG_W(ctx, "Unknown replication" + << ": rid# " << rid); + return true; + } + + NIceDb::TNiceDb db(txc.DB); + + if (Ev->Get()->IsSuccess()) { + for (const auto& target : Ev->Get()->ToAdd) { + const auto kind = TargetKindFromEntryType(target.first.Type); + const auto& srcPath = target.first.Name; + const auto& dstPath = target.second; + + const auto tid = Replication->AddTarget(kind, srcPath, dstPath); + db.Table<Schema::Targets>().Key(rid, tid).Update( + NIceDb::TUpdate<Schema::Targets::Kind>(kind), + NIceDb::TUpdate<Schema::Targets::SrcPath>(srcPath), + NIceDb::TUpdate<Schema::Targets::DstPath>(dstPath) + ); + + CLOG_N(ctx, "Add target" + << ": rid# " << rid + << ", tid# " << tid + << ", kind# " << kind + << ", srcPath# " << srcPath + << ", dstPath# " << dstPath); + } + } else { + const auto error = JoinSeq(", ", Ev->Get()->Failed); + Replication->SetState(TReplication::EState::Error, TStringBuilder() << "Discovery error: " << error); + + CLOG_E(ctx, "Discovery error" + << ": rid# " << rid + << ", error# " << error); + } + + db.Table<Schema::Replications>().Key(Replication->GetId()).Update( + NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()), + NIceDb::TUpdate<Schema::Replications::Issue>(Replication->GetIssue()), + NIceDb::TUpdate<Schema::Replications::NextTargetId>(Replication->GetNextTargetId()) + ); + + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxDiscoveryResult + +void TController::RunTxDiscoveryResult(TEvPrivate::TEvDiscoveryResult::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxDiscoveryResult(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_drop_replication.cpp b/ydb/core/tx/replication/controller/tx_drop_replication.cpp index d5d89b195ef..3040915dce7 100644 --- a/ydb/core/tx/replication/controller/tx_drop_replication.cpp +++ b/ydb/core/tx/replication/controller/tx_drop_replication.cpp @@ -1,77 +1,77 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxDropReplication: public TTxBase { - TEvController::TEvDropReplication::TPtr Ev; - THolder<TEvController::TEvDropReplicationResult> Result; - TReplication::TPtr Replication; - -public: - explicit TTxDropReplication(TController* self, TEvController::TEvDropReplication::TPtr& ev) - : TTxBase("TxDropReplication", self) - , Ev(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_DROP_REPLICATION; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); - - const auto& record = Ev->Get()->Record; - Result = MakeHolder<TEvController::TEvDropReplicationResult>(); - Result->Record.MutableOperationId()->CopyFrom(record.GetOperationId()); - Result->Record.SetOrigin(Self->TabletID()); - - const auto pathId = PathIdFromPathId(record.GetPathId()); - Replication = Self->Find(pathId); - - if (!Replication) { - CLOG_W(ctx, "Cannot drop unknown replication" - << ": pathId# " << pathId); - - Result->Record.SetStatus(NKikimrReplication::TEvDropReplicationResult::NOT_FOUND); - return true; - } - - NIceDb::TNiceDb db(txc.DB); - Replication->SetState(TReplication::EState::Removing); - db.Table<Schema::Replications>().Key(Replication->GetId()).Update( - NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()) - ); - - CLOG_N(ctx, "Drop replication" - << ": rid# " << Replication->GetId() - << ", pathId# " << pathId); - - // TODO: delay response - Result->Record.SetStatus(NKikimrReplication::TEvDropReplicationResult::SUCCESS); - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - - if (Result) { - ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); - } - - if (Replication) { - Replication->Progress(ctx); - } - } - -}; // TTxDropReplication - -void TController::RunTxDropReplication(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxDropReplication(this, ev), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxDropReplication: public TTxBase { + TEvController::TEvDropReplication::TPtr Ev; + THolder<TEvController::TEvDropReplicationResult> Result; + TReplication::TPtr Replication; + +public: + explicit TTxDropReplication(TController* self, TEvController::TEvDropReplication::TPtr& ev) + : TTxBase("TxDropReplication", self) + , Ev(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_DROP_REPLICATION; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute: " << Ev->Get()->ToString()); + + const auto& record = Ev->Get()->Record; + Result = MakeHolder<TEvController::TEvDropReplicationResult>(); + Result->Record.MutableOperationId()->CopyFrom(record.GetOperationId()); + Result->Record.SetOrigin(Self->TabletID()); + + const auto pathId = PathIdFromPathId(record.GetPathId()); + Replication = Self->Find(pathId); + + if (!Replication) { + CLOG_W(ctx, "Cannot drop unknown replication" + << ": pathId# " << pathId); + + Result->Record.SetStatus(NKikimrReplication::TEvDropReplicationResult::NOT_FOUND); + return true; + } + + NIceDb::TNiceDb db(txc.DB); + Replication->SetState(TReplication::EState::Removing); + db.Table<Schema::Replications>().Key(Replication->GetId()).Update( + NIceDb::TUpdate<Schema::Replications::State>(Replication->GetState()) + ); + + CLOG_N(ctx, "Drop replication" + << ": rid# " << Replication->GetId() + << ", pathId# " << pathId); + + // TODO: delay response + Result->Record.SetStatus(NKikimrReplication::TEvDropReplicationResult::SUCCESS); + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + + if (Result) { + ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); + } + + if (Replication) { + Replication->Progress(ctx); + } + } + +}; // TTxDropReplication + +void TController::RunTxDropReplication(TEvController::TEvDropReplication::TPtr& ev, const TActorContext& ctx) { + Execute(new TTxDropReplication(this, ev), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_init.cpp b/ydb/core/tx/replication/controller/tx_init.cpp index 434e2024855..a0e234fa4b4 100644 --- a/ydb/core/tx/replication/controller/tx_init.cpp +++ b/ydb/core/tx/replication/controller/tx_init.cpp @@ -1,179 +1,179 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxInit: public TTxBase { - template <typename TRowset> - class TSysParamLoader: public ISysParamLoader { - public: - explicit TSysParamLoader(TRowset& rowset) - : Rowset(rowset) - { - } - - ui64 LoadInt() { return Rowset.template GetValue<Schema::SysParams::IntValue>(); } - TString LoadText() { return Rowset.template GetValue<Schema::SysParams::TextValue>(); } - TString LoadBinary() { return Rowset.template GetValue<Schema::SysParams::BinaryValue>(); } - - private: - TRowset& Rowset; - }; - - bool LoadSysParams(NIceDb::TNiceDb& db) { - auto rowset = db.Table<Schema::SysParams>().Select(); - if (!rowset.IsReady()) { - return false; - } - - auto loader = MakeHolder<TSysParamLoader<decltype(rowset)>>(rowset); - while (!rowset.EndOfSet()) { - Self->SysParams.Load(rowset.GetValue<Schema::SysParams::Id>(), loader.Get()); - if (!rowset.Next()) { - return false; - } - } - - return true; - } - - bool LoadReplications(NIceDb::TNiceDb& db) { - auto rowset = db.Table<Schema::Replications>().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const auto rid = rowset.GetValue<Schema::Replications::Id>(); - const auto pathId = TPathId( - rowset.GetValue<Schema::Replications::PathOwnerId>(), - rowset.GetValue<Schema::Replications::PathLocalId>() - ); - const auto config = rowset.GetValue<Schema::Replications::Config>(); - const auto state = rowset.GetValue<Schema::Replications::State>(); - const auto issue = rowset.GetValue<Schema::Replications::Issue>(); - const auto nextTid = rowset.GetValue<Schema::Replications::NextTargetId>(); - - auto replication = Self->Add(rid, pathId, config); - replication->SetState(state, issue); - replication->SetNextTargetId(nextTid); - - if (!rowset.Next()) { - return false; - } - } - - return true; - } - - bool LoadTargets(NIceDb::TNiceDb& db) { - auto rowset = db.Table<Schema::Targets>().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const auto rid = rowset.GetValue<Schema::Targets::ReplicationId>(); - const auto tid = rowset.GetValue<Schema::Targets::Id>(); - const auto kind = rowset.GetValue<Schema::Targets::Kind>(); - const auto srcPath = rowset.GetValue<Schema::Targets::SrcPath>(); - const auto dstPath = rowset.GetValue<Schema::Targets::DstPath>(); - const auto dstState = rowset.GetValue<Schema::Targets::DstState>(); - const auto issue = rowset.GetValue<Schema::Targets::Issue>(); - const auto dstPathId = TPathId( - rowset.GetValue<Schema::Targets::DstPathOwnerId>(), - rowset.GetValue<Schema::Targets::DstPathLocalId>() - ); - - auto replication = Self->Find(rid); - Y_VERIFY_S(replication, "Unknown replication: " << rid); - - auto* target = replication->AddTarget(tid, kind, srcPath, dstPath); - Y_VERIFY(target); - - target->SetDstState(dstState); - target->SetDstPathId(dstPathId); - target->SetIssue(issue); - - if (!rowset.Next()) { - return false; - } - } - - return true; - } - - bool LoadSrcStreams(NIceDb::TNiceDb& db) { - auto rowset = db.Table<Schema::SrcStreams>().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const auto rid = rowset.GetValue<Schema::SrcStreams::ReplicationId>(); - const auto tid = rowset.GetValue<Schema::SrcStreams::TargetId>(); - const auto name = rowset.GetValue<Schema::SrcStreams::Name>(); - const auto state = rowset.GetValue<Schema::SrcStreams::State>(); - - auto replication = Self->Find(rid); - Y_VERIFY_S(replication, "Unknown replication: " << rid); - - auto* target = replication->FindTarget(tid); - Y_VERIFY_S(target, "Unknown target" - << ": rid# " << rid - << ", tid# " << tid); - - target->SetStreamName(name); - target->SetStreamState(state); - - if (!rowset.Next()) { - return false; - } - } - - return true; - } - - inline bool Load(NIceDb::TNiceDb& db) { - Self->Reset(); - return LoadSysParams(db) - && LoadReplications(db) - && LoadTargets(db) - && LoadSrcStreams(db); - } - - inline bool Load(NTable::TDatabase& toughDb) { - NIceDb::TNiceDb db(toughDb); - return Load(db); - } - -public: - explicit TTxInit(TSelf* self) - : TTxBase("TxInit", self) - { - } - - TTxType GetTxType() const override { - return TXTYPE_INIT; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute"); - return Load(txc.DB); - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - Self->SwitchToWork(ctx); - } - -}; // TTxInit - -void TController::RunTxInit(const TActorContext& ctx) { - Execute(new TTxInit(this), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxInit: public TTxBase { + template <typename TRowset> + class TSysParamLoader: public ISysParamLoader { + public: + explicit TSysParamLoader(TRowset& rowset) + : Rowset(rowset) + { + } + + ui64 LoadInt() { return Rowset.template GetValue<Schema::SysParams::IntValue>(); } + TString LoadText() { return Rowset.template GetValue<Schema::SysParams::TextValue>(); } + TString LoadBinary() { return Rowset.template GetValue<Schema::SysParams::BinaryValue>(); } + + private: + TRowset& Rowset; + }; + + bool LoadSysParams(NIceDb::TNiceDb& db) { + auto rowset = db.Table<Schema::SysParams>().Select(); + if (!rowset.IsReady()) { + return false; + } + + auto loader = MakeHolder<TSysParamLoader<decltype(rowset)>>(rowset); + while (!rowset.EndOfSet()) { + Self->SysParams.Load(rowset.GetValue<Schema::SysParams::Id>(), loader.Get()); + if (!rowset.Next()) { + return false; + } + } + + return true; + } + + bool LoadReplications(NIceDb::TNiceDb& db) { + auto rowset = db.Table<Schema::Replications>().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const auto rid = rowset.GetValue<Schema::Replications::Id>(); + const auto pathId = TPathId( + rowset.GetValue<Schema::Replications::PathOwnerId>(), + rowset.GetValue<Schema::Replications::PathLocalId>() + ); + const auto config = rowset.GetValue<Schema::Replications::Config>(); + const auto state = rowset.GetValue<Schema::Replications::State>(); + const auto issue = rowset.GetValue<Schema::Replications::Issue>(); + const auto nextTid = rowset.GetValue<Schema::Replications::NextTargetId>(); + + auto replication = Self->Add(rid, pathId, config); + replication->SetState(state, issue); + replication->SetNextTargetId(nextTid); + + if (!rowset.Next()) { + return false; + } + } + + return true; + } + + bool LoadTargets(NIceDb::TNiceDb& db) { + auto rowset = db.Table<Schema::Targets>().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const auto rid = rowset.GetValue<Schema::Targets::ReplicationId>(); + const auto tid = rowset.GetValue<Schema::Targets::Id>(); + const auto kind = rowset.GetValue<Schema::Targets::Kind>(); + const auto srcPath = rowset.GetValue<Schema::Targets::SrcPath>(); + const auto dstPath = rowset.GetValue<Schema::Targets::DstPath>(); + const auto dstState = rowset.GetValue<Schema::Targets::DstState>(); + const auto issue = rowset.GetValue<Schema::Targets::Issue>(); + const auto dstPathId = TPathId( + rowset.GetValue<Schema::Targets::DstPathOwnerId>(), + rowset.GetValue<Schema::Targets::DstPathLocalId>() + ); + + auto replication = Self->Find(rid); + Y_VERIFY_S(replication, "Unknown replication: " << rid); + + auto* target = replication->AddTarget(tid, kind, srcPath, dstPath); + Y_VERIFY(target); + + target->SetDstState(dstState); + target->SetDstPathId(dstPathId); + target->SetIssue(issue); + + if (!rowset.Next()) { + return false; + } + } + + return true; + } + + bool LoadSrcStreams(NIceDb::TNiceDb& db) { + auto rowset = db.Table<Schema::SrcStreams>().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const auto rid = rowset.GetValue<Schema::SrcStreams::ReplicationId>(); + const auto tid = rowset.GetValue<Schema::SrcStreams::TargetId>(); + const auto name = rowset.GetValue<Schema::SrcStreams::Name>(); + const auto state = rowset.GetValue<Schema::SrcStreams::State>(); + + auto replication = Self->Find(rid); + Y_VERIFY_S(replication, "Unknown replication: " << rid); + + auto* target = replication->FindTarget(tid); + Y_VERIFY_S(target, "Unknown target" + << ": rid# " << rid + << ", tid# " << tid); + + target->SetStreamName(name); + target->SetStreamState(state); + + if (!rowset.Next()) { + return false; + } + } + + return true; + } + + inline bool Load(NIceDb::TNiceDb& db) { + Self->Reset(); + return LoadSysParams(db) + && LoadReplications(db) + && LoadTargets(db) + && LoadSrcStreams(db); + } + + inline bool Load(NTable::TDatabase& toughDb) { + NIceDb::TNiceDb db(toughDb); + return Load(db); + } + +public: + explicit TTxInit(TSelf* self) + : TTxBase("TxInit", self) + { + } + + TTxType GetTxType() const override { + return TXTYPE_INIT; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute"); + return Load(txc.DB); + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + Self->SwitchToWork(ctx); + } + +}; // TTxInit + +void TController::RunTxInit(const TActorContext& ctx) { + Execute(new TTxInit(this), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/tx_init_schema.cpp b/ydb/core/tx/replication/controller/tx_init_schema.cpp index 4ec78c97c23..20f50ab2f24 100644 --- a/ydb/core/tx/replication/controller/tx_init_schema.cpp +++ b/ydb/core/tx/replication/controller/tx_init_schema.cpp @@ -1,40 +1,40 @@ -#include "controller_impl.h" - -namespace NKikimr { -namespace NReplication { -namespace NController { - -class TController::TTxInitSchema: public TTxBase { -public: - explicit TTxInitSchema(TController* self) - : TTxBase("TxInitSchema", self) - { - } - - TTxType GetTxType() const override { - return TXTYPE_INIT_SCHEMA; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - CLOG_D(ctx, "Execute"); - - NIceDb::TNiceDb db(txc.DB); - db.Materialize<Schema>(); - - return true; - } - - void Complete(const TActorContext& ctx) override { - CLOG_D(ctx, "Complete"); - Self->RunTxInit(ctx); - } - -}; // TTxInitSchema - -void TController::RunTxInitSchema(const TActorContext& ctx) { - Execute(new TTxInitSchema(this), ctx); -} - -} // NController -} // NReplication -} // NKikimr +#include "controller_impl.h" + +namespace NKikimr { +namespace NReplication { +namespace NController { + +class TController::TTxInitSchema: public TTxBase { +public: + explicit TTxInitSchema(TController* self) + : TTxBase("TxInitSchema", self) + { + } + + TTxType GetTxType() const override { + return TXTYPE_INIT_SCHEMA; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + CLOG_D(ctx, "Execute"); + + NIceDb::TNiceDb db(txc.DB); + db.Materialize<Schema>(); + + return true; + } + + void Complete(const TActorContext& ctx) override { + CLOG_D(ctx, "Complete"); + Self->RunTxInit(ctx); + } + +}; // TTxInitSchema + +void TController::RunTxInitSchema(const TActorContext& ctx) { + Execute(new TTxInitSchema(this), ctx); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/util.h b/ydb/core/tx/replication/controller/util.h index 193b67607da..c4162a26dbd 100644 --- a/ydb/core/tx/replication/controller/util.h +++ b/ydb/core/tx/replication/controller/util.h @@ -1,81 +1,81 @@ -#pragma once - -#include "replication.h" - +#pragma once + +#include "replication.h" + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> -#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - -#include <ydb/core/util/yverify_stream.h> - -#include <util/generic/algorithm.h> -#include <util/generic/maybe.h> -#include <util/generic/size_literals.h> - -namespace NKikimr { -namespace NReplication { -namespace NController { - -inline TMaybe<TReplication::ETargetKind> TryTargetKindFromEntryType(NYdb::NScheme::ESchemeEntryType type) { - switch (type) { - case NYdb::NScheme::ESchemeEntryType::Table: - return TReplication::ETargetKind::Table; - case NYdb::NScheme::ESchemeEntryType::Unknown: - case NYdb::NScheme::ESchemeEntryType::Directory: - case NYdb::NScheme::ESchemeEntryType::PqGroup: - case NYdb::NScheme::ESchemeEntryType::SubDomain: - case NYdb::NScheme::ESchemeEntryType::RtmrVolume: - case NYdb::NScheme::ESchemeEntryType::BlockStoreVolume: - case NYdb::NScheme::ESchemeEntryType::CoordinationNode: - case NYdb::NScheme::ESchemeEntryType::Sequence: - case NYdb::NScheme::ESchemeEntryType::Replication: - return Nothing(); - } -} - -inline TReplication::ETargetKind TargetKindFromEntryType(NYdb::NScheme::ESchemeEntryType type) { - auto res = TryTargetKindFromEntryType(type); - Y_VERIFY_S(res, "Unexpected entry type: " << static_cast<i32>(type)); - return *res; -} - -inline TString& TruncatedIssue(TString& issue) { - static constexpr ui32 sizeLimit = 2_KB; - static constexpr TStringBuf ellipsis = "..."; - - if (issue.size() > sizeLimit) { - issue.resize(sizeLimit - ellipsis.size()); - issue.append(ellipsis); - } - - return issue; -} - -inline auto DefaultRetryableErrors() { - using EStatus = NYdb::EStatus; - return TVector<EStatus>{ - EStatus::ABORTED, - EStatus::UNAVAILABLE, - EStatus::OVERLOADED, - EStatus::TIMEOUT, - EStatus::BAD_SESSION, - EStatus::SESSION_EXPIRED, - EStatus::CANCELLED, - EStatus::UNDETERMINED, - EStatus::SESSION_BUSY, - EStatus::CLIENT_DISCOVERY_FAILED, - EStatus::CLIENT_LIMITS_REACHED, - }; -} - -inline bool IsRetryableError(const NYdb::TStatus status, const TVector<NYdb::EStatus>& retryable) { - return status.IsTransportError() || Find(retryable, status.GetStatus()) != retryable.end(); -} - -inline bool IsRetryableError(const NYdb::TStatus status) { - static auto defaultRetryableErrors = DefaultRetryableErrors(); - return IsRetryableError(status, defaultRetryableErrors); -} - -} // NController -} // NReplication -} // NKikimr +#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> + +#include <ydb/core/util/yverify_stream.h> + +#include <util/generic/algorithm.h> +#include <util/generic/maybe.h> +#include <util/generic/size_literals.h> + +namespace NKikimr { +namespace NReplication { +namespace NController { + +inline TMaybe<TReplication::ETargetKind> TryTargetKindFromEntryType(NYdb::NScheme::ESchemeEntryType type) { + switch (type) { + case NYdb::NScheme::ESchemeEntryType::Table: + return TReplication::ETargetKind::Table; + case NYdb::NScheme::ESchemeEntryType::Unknown: + case NYdb::NScheme::ESchemeEntryType::Directory: + case NYdb::NScheme::ESchemeEntryType::PqGroup: + case NYdb::NScheme::ESchemeEntryType::SubDomain: + case NYdb::NScheme::ESchemeEntryType::RtmrVolume: + case NYdb::NScheme::ESchemeEntryType::BlockStoreVolume: + case NYdb::NScheme::ESchemeEntryType::CoordinationNode: + case NYdb::NScheme::ESchemeEntryType::Sequence: + case NYdb::NScheme::ESchemeEntryType::Replication: + return Nothing(); + } +} + +inline TReplication::ETargetKind TargetKindFromEntryType(NYdb::NScheme::ESchemeEntryType type) { + auto res = TryTargetKindFromEntryType(type); + Y_VERIFY_S(res, "Unexpected entry type: " << static_cast<i32>(type)); + return *res; +} + +inline TString& TruncatedIssue(TString& issue) { + static constexpr ui32 sizeLimit = 2_KB; + static constexpr TStringBuf ellipsis = "..."; + + if (issue.size() > sizeLimit) { + issue.resize(sizeLimit - ellipsis.size()); + issue.append(ellipsis); + } + + return issue; +} + +inline auto DefaultRetryableErrors() { + using EStatus = NYdb::EStatus; + return TVector<EStatus>{ + EStatus::ABORTED, + EStatus::UNAVAILABLE, + EStatus::OVERLOADED, + EStatus::TIMEOUT, + EStatus::BAD_SESSION, + EStatus::SESSION_EXPIRED, + EStatus::CANCELLED, + EStatus::UNDETERMINED, + EStatus::SESSION_BUSY, + EStatus::CLIENT_DISCOVERY_FAILED, + EStatus::CLIENT_LIMITS_REACHED, + }; +} + +inline bool IsRetryableError(const NYdb::TStatus status, const TVector<NYdb::EStatus>& retryable) { + return status.IsTransportError() || Find(retryable, status.GetStatus()) != retryable.end(); +} + +inline bool IsRetryableError(const NYdb::TStatus status) { + static auto defaultRetryableErrors = DefaultRetryableErrors(); + return IsRetryableError(status, defaultRetryableErrors); +} + +} // NController +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/controller/ya.make b/ydb/core/tx/replication/controller/ya.make index 01cfae6eafa..6ff2fb83e1f 100644 --- a/ydb/core/tx/replication/controller/ya.make +++ b/ydb/core/tx/replication/controller/ya.make @@ -1,44 +1,44 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( ydb/core/base ydb/core/engine/minikql ydb/core/protos ydb/core/tablet_flat - ydb/core/tx/replication/ydb_proxy - ydb/core/util - ydb/core/ydb_convert -) - -SRCS( - controller.cpp - discoverer.cpp - dst_creator.cpp - logging.cpp - private_events.cpp - replication.cpp - stream_creator.cpp - sys_params.cpp - target_base.cpp - target_table.cpp - target_with_stream.cpp - tx_assign_stream_name.cpp - tx_create_dst_result.cpp - tx_create_replication.cpp - tx_create_stream_result.cpp - tx_discovery_result.cpp - tx_drop_replication.cpp - tx_init.cpp - tx_init_schema.cpp -) - -GENERATE_ENUM_SERIALIZATION(replication.h) - -YQL_LAST_ABI_VERSION() - -END() + ydb/core/tx/replication/ydb_proxy + ydb/core/util + ydb/core/ydb_convert +) + +SRCS( + controller.cpp + discoverer.cpp + dst_creator.cpp + logging.cpp + private_events.cpp + replication.cpp + stream_creator.cpp + sys_params.cpp + target_base.cpp + target_table.cpp + target_with_stream.cpp + tx_assign_stream_name.cpp + tx_create_dst_result.cpp + tx_create_replication.cpp + tx_create_stream_result.cpp + tx_discovery_result.cpp + tx_drop_replication.cpp + tx_init.cpp + tx_init_schema.cpp +) + +GENERATE_ENUM_SERIALIZATION(replication.h) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/replication/ut/ya.make b/ydb/core/tx/replication/ut/ya.make index b26617f0ed0..7130cde31b1 100644 --- a/ydb/core/tx/replication/ut/ya.make +++ b/ydb/core/tx/replication/ut/ya.make @@ -1,8 +1,8 @@ -OWNER( - ilnaz - g:kikimr -) - -RECURSE( - ../ydb_proxy/ut -) +OWNER( + ilnaz + g:kikimr +) + +RECURSE( + ../ydb_proxy/ut +) diff --git a/ydb/core/tx/replication/ya.make b/ydb/core/tx/replication/ya.make index da418d9416c..6707c867dc9 100644 --- a/ydb/core/tx/replication/ya.make +++ b/ydb/core/tx/replication/ya.make @@ -1,13 +1,13 @@ -OWNER( - ilnaz - g:kikimr -) - -RECURSE( - controller - ydb_proxy -) - -RECURSE_FOR_TESTS( - ut -) +OWNER( + ilnaz + g:kikimr +) + +RECURSE( + controller + ydb_proxy +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/tx/replication/ydb_proxy/ut/ya.make b/ydb/core/tx/replication/ydb_proxy/ut/ya.make index 73e5635e5cd..605ac66a6ae 100644 --- a/ydb/core/tx/replication/ydb_proxy/ut/ya.make +++ b/ydb/core/tx/replication/ydb_proxy/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/core/tx/replication/ydb_proxy) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( - library/cpp/testing/unittest + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( + library/cpp/testing/unittest ydb/core/testlib -) - -SRCS( - ydb_proxy_ut.cpp -) - -YQL_LAST_ABI_VERSION() - -END() +) + +SRCS( + ydb_proxy_ut.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/replication/ydb_proxy/ya.make b/ydb/core/tx/replication/ydb_proxy/ya.make index b8af7da454e..88bf0c911a2 100644 --- a/ydb/core/tx/replication/ydb_proxy/ya.make +++ b/ydb/core/tx/replication/ydb_proxy/ya.make @@ -1,27 +1,27 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( - ydb/core/base +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( + ydb/core/base ydb/core/protos ydb/public/sdk/cpp/client/ydb_driver ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_table ydb/public/sdk/cpp/client/ydb_types/credentials -) - -SRCS( - ydb_proxy.cpp -) - -YQL_LAST_ABI_VERSION() - -END() - -RECURSE_FOR_TESTS( - ut -) +) + +SRCS( + ydb_proxy.cpp +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp index 26691933d43..45de477068c 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp @@ -1,319 +1,319 @@ -#include "ydb_proxy.h" - +#include "ydb_proxy.h" + #include <ydb/core/protos/replication.pb.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <ydb/core/base/appdata.h> - -#include <util/generic/hash_set.h> - -#include <memory> -#include <mutex> - -namespace NKikimr { -namespace NReplication { - -using namespace NKikimrReplication; -using namespace NYdb; -using namespace NYdb::NScheme; -using namespace NYdb::NTable; - -template <typename TDerived> -class TBaseProxyActor: public TActor<TDerived> { - class TRequest; - using TRequestPtr = std::shared_ptr<TRequest>; - - struct TEvPrivate { - enum EEv { - EvComplete = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); - - struct TEvComplete: public TEventLocal<TEvComplete, EvComplete> { - const TRequestPtr Request; - - explicit TEvComplete(const TRequestPtr& request) - : Request(request) - { - } - }; - - }; // TEvPrivate - - class TRequest: public std::enable_shared_from_this<TRequest> { - friend class TBaseProxyActor<TDerived>; - - public: - explicit TRequest(const TActorSystem* sys, const TActorId& self, const TActorId& sender, ui64 cookie) - : ActorSystem(sys) - , Self(self) - , Sender(sender) - , Cookie(cookie) - { - } - - void Complete(IEventBase* ev) { - Send(Sender, ev, Cookie); - Send(Self, new typename TEvPrivate::TEvComplete(this->shared_from_this())); - } - - private: - void Send(const TActorId& recipient, IEventBase* ev, ui64 cookie = 0) { - std::lock_guard<std::mutex> lock(RWActorSystem); - - if (ActorSystem) { - ActorSystem->Send(new IEventHandle(recipient, Self, ev, 0, cookie)); - } - } - - void ClearActorSystem() { - std::lock_guard<std::mutex> lock(RWActorSystem); - ActorSystem = nullptr; - } - - private: - const TActorSystem* ActorSystem; - const TActorId Self; - const TActorId Sender; - const ui64 Cookie; - - std::mutex RWActorSystem; - - }; // TRequest - - struct TRequestPtrHash { - Y_FORCE_INLINE size_t operator()(const TRequestPtr& ptr) const { - return THash<TRequest*>()(ptr.get()); - } - }; - - void Handle(typename TEvPrivate::TEvComplete::TPtr& ev) { - Requests.erase(ev->Get()->Request); - } - - void PassAway() override { - Requests.clear(); - IActor::PassAway(); - } - -protected: - using TActor<TDerived>::TActor; - - virtual ~TBaseProxyActor() { - for (auto& request : Requests) { - request->ClearActorSystem(); - } - } - - std::weak_ptr<TRequest> MakeRequest(const TActorId& sender, ui64 cookie) { - auto request = std::make_shared<TRequest>(TlsActivationContext->ActorSystem(), this->SelfId(), sender, cookie); - Requests.emplace(request); - return request; - } - - template <typename TEvResponse> - static void Complete(std::weak_ptr<TRequest> request, const typename TEvResponse::TResult& result) { - if (auto r = request.lock()) { - r->Complete(new TEvResponse(result)); - } - } - - template <typename TEvResponse> - static auto CreateCallback(std::weak_ptr<TRequest> request) { - return [request](const typename TEvResponse::TAsyncResult& result) { - Complete<TEvResponse>(request, result.GetValueSync()); - }; - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvPrivate::TEvComplete, Handle); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - THashSet<TRequestPtr, TRequestPtrHash> Requests; - -}; // TBaseProxyActor - -class TYdbProxy: public TBaseProxyActor<TYdbProxy> { - template <typename TEvResponse, typename TClient, typename... Args> - using TFunc = typename TEvResponse::TAsyncResult(TClient::*)(Args...); - - template <typename TClient> - TClient* EnsureClient(THolder<TClient>& client) { - if (!client) { - Y_VERIFY(AppData()->YdbDriver); - client.Reset(new TClient(*AppData()->YdbDriver, Settings)); - } - - return client.Get(); - } - - template <typename TClient> - TClient* EnsureClient() { - if constexpr (std::is_same_v<TClient, TSchemeClient>) { - return EnsureClient<TClient>(SchemeClient); - } else if constexpr (std::is_same_v<TClient, TTableClient>) { - return EnsureClient<TClient>(TableClient); - } else { - Y_FAIL("unreachable"); - } - } - - template <typename TEvResponse, typename TEvRequestPtr, typename TClient, typename... Args> - void Call(TEvRequestPtr& ev, TFunc<TEvResponse, TClient, Args...> func) { - auto* client = EnsureClient<TClient>(); - auto request = MakeRequest(ev->Sender, ev->Cookie); - auto args = std::move(ev->Get()->GetArgs()); - auto cb = CreateCallback<TEvResponse>(request); - - std::apply(func, std::tuple_cat(std::tie(client), std::move(args))).Subscribe(std::move(cb)); - } - - template <typename TEvResponse, typename TEvRequestPtr, typename TSession, typename... Args> - void CallSession(TEvRequestPtr& ev, TFunc<TEvResponse, TSession, Args...> func) { - auto* client = EnsureClient<TTableClient>(); - auto request = MakeRequest(ev->Sender, ev->Cookie); - auto args = std::move(ev->Get()->GetArgs()); - auto cb = [request, func, args = std::move(args)](const TAsyncCreateSessionResult& result) { - auto sessionResult = result.GetValueSync(); - if (!sessionResult.IsSuccess()) { - return Complete<TEvYdbProxy::TEvCreateSessionResponse>(request, sessionResult); - } - - auto session = sessionResult.GetSession(); - auto cb = CreateCallback<TEvResponse>(request); - std::apply(func, std::tuple_cat(std::tie(session), std::move(args))).Subscribe(std::move(cb)); - }; - - client->GetSession().Subscribe(std::move(cb)); - } - - void Handle(TEvYdbProxy::TEvMakeDirectoryRequest::TPtr& ev) { - Call<TEvYdbProxy::TEvMakeDirectoryResponse>(ev, &TSchemeClient::MakeDirectory); - } - - void Handle(TEvYdbProxy::TEvRemoveDirectoryRequest::TPtr& ev) { - Call<TEvYdbProxy::TEvRemoveDirectoryResponse>(ev, &TSchemeClient::RemoveDirectory); - } - - void Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { - Call<TEvYdbProxy::TEvDescribePathResponse>(ev, &TSchemeClient::DescribePath); - } - - void Handle(TEvYdbProxy::TEvListDirectoryRequest::TPtr& ev) { - Call<TEvYdbProxy::TEvListDirectoryResponse>(ev, &TSchemeClient::ListDirectory); - } - - void Handle(TEvYdbProxy::TEvModifyPermissionsRequest::TPtr& ev) { - Call<TEvYdbProxy::TEvModifyPermissionsResponse>(ev, &TSchemeClient::ModifyPermissions); - } - - void Handle(TEvYdbProxy::TEvCreateTableRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvCreateTableResponse>(ev, &TSession::CreateTable); - } - - void Handle(TEvYdbProxy::TEvDropTableRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvDropTableResponse>(ev, &TSession::DropTable); - } - - void Handle(TEvYdbProxy::TEvAlterTableRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvAlterTableResponse>(ev, &TSession::AlterTable); - } - - void Handle(TEvYdbProxy::TEvCopyTableRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvCopyTableResponse>(ev, &TSession::CopyTable); - } - - void Handle(TEvYdbProxy::TEvCopyTablesRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvCopyTablesResponse>(ev, &TSession::CopyTables); - } - - void Handle(TEvYdbProxy::TEvRenameTablesRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvRenameTablesResponse>(ev, &TSession::RenameTables); - } - - void Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { - CallSession<TEvYdbProxy::TEvDescribeTableResponse>(ev, &TSession::DescribeTable); - } - - static TClientSettings MakeSettings(const TString& endpoint, const TString& database) { - return TClientSettings() - .DiscoveryEndpoint(endpoint) - .DiscoveryMode(EDiscoveryMode::Async) - .Database(database); - } - - static TClientSettings MakeSettings(const TString& endpoint, const TString& database, const TString& token) { - return MakeSettings(endpoint, database) - .AuthToken(token); - } - - static TClientSettings MakeSettings(const TString& endpoint, const TString& database, const TStaticCredentials& credentials) { - return MakeSettings(endpoint, database) - .CredentialsProviderFactory(CreateLoginCredentialsProviderFactory({ - .User = credentials.GetUser(), - .Password = credentials.GetPassword(), - })); - } - -public: - template <typename... Args> - explicit TYdbProxy(Args&&... args) - : TBaseProxyActor(&TThis::StateWork) - , Settings(MakeSettings(std::forward<Args>(args)...)) - { - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - // Scheme - hFunc(TEvYdbProxy::TEvMakeDirectoryRequest, Handle); - hFunc(TEvYdbProxy::TEvRemoveDirectoryRequest, Handle); - hFunc(TEvYdbProxy::TEvDescribePathRequest, Handle); - hFunc(TEvYdbProxy::TEvListDirectoryRequest, Handle); - hFunc(TEvYdbProxy::TEvModifyPermissionsRequest, Handle); - // Table - hFunc(TEvYdbProxy::TEvCreateTableRequest, Handle); - hFunc(TEvYdbProxy::TEvDropTableRequest, Handle); - hFunc(TEvYdbProxy::TEvAlterTableRequest, Handle); - hFunc(TEvYdbProxy::TEvCopyTableRequest, Handle); - hFunc(TEvYdbProxy::TEvCopyTablesRequest, Handle); - hFunc(TEvYdbProxy::TEvRenameTablesRequest, Handle); - hFunc(TEvYdbProxy::TEvDescribeTableRequest, Handle); - - default: - return StateBase(ev, TlsActivationContext->AsActorContext()); - } - } - -private: - const TClientSettings Settings; - THolder<TSchemeClient> SchemeClient; - THolder<TTableClient> TableClient; - -}; // TYdbProxy - -IActor* CreateYdbProxy(const TString& endpoint, const TString& database) { - return new TYdbProxy(endpoint, database); -} - -IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TString& token) { - return new TYdbProxy(endpoint, database, token); -} - -IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TStaticCredentials& credentials) { - return new TYdbProxy(endpoint, database, credentials); -} - -} // NReplication -} // NKikimr + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <ydb/core/base/appdata.h> + +#include <util/generic/hash_set.h> + +#include <memory> +#include <mutex> + +namespace NKikimr { +namespace NReplication { + +using namespace NKikimrReplication; +using namespace NYdb; +using namespace NYdb::NScheme; +using namespace NYdb::NTable; + +template <typename TDerived> +class TBaseProxyActor: public TActor<TDerived> { + class TRequest; + using TRequestPtr = std::shared_ptr<TRequest>; + + struct TEvPrivate { + enum EEv { + EvComplete = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + struct TEvComplete: public TEventLocal<TEvComplete, EvComplete> { + const TRequestPtr Request; + + explicit TEvComplete(const TRequestPtr& request) + : Request(request) + { + } + }; + + }; // TEvPrivate + + class TRequest: public std::enable_shared_from_this<TRequest> { + friend class TBaseProxyActor<TDerived>; + + public: + explicit TRequest(const TActorSystem* sys, const TActorId& self, const TActorId& sender, ui64 cookie) + : ActorSystem(sys) + , Self(self) + , Sender(sender) + , Cookie(cookie) + { + } + + void Complete(IEventBase* ev) { + Send(Sender, ev, Cookie); + Send(Self, new typename TEvPrivate::TEvComplete(this->shared_from_this())); + } + + private: + void Send(const TActorId& recipient, IEventBase* ev, ui64 cookie = 0) { + std::lock_guard<std::mutex> lock(RWActorSystem); + + if (ActorSystem) { + ActorSystem->Send(new IEventHandle(recipient, Self, ev, 0, cookie)); + } + } + + void ClearActorSystem() { + std::lock_guard<std::mutex> lock(RWActorSystem); + ActorSystem = nullptr; + } + + private: + const TActorSystem* ActorSystem; + const TActorId Self; + const TActorId Sender; + const ui64 Cookie; + + std::mutex RWActorSystem; + + }; // TRequest + + struct TRequestPtrHash { + Y_FORCE_INLINE size_t operator()(const TRequestPtr& ptr) const { + return THash<TRequest*>()(ptr.get()); + } + }; + + void Handle(typename TEvPrivate::TEvComplete::TPtr& ev) { + Requests.erase(ev->Get()->Request); + } + + void PassAway() override { + Requests.clear(); + IActor::PassAway(); + } + +protected: + using TActor<TDerived>::TActor; + + virtual ~TBaseProxyActor() { + for (auto& request : Requests) { + request->ClearActorSystem(); + } + } + + std::weak_ptr<TRequest> MakeRequest(const TActorId& sender, ui64 cookie) { + auto request = std::make_shared<TRequest>(TlsActivationContext->ActorSystem(), this->SelfId(), sender, cookie); + Requests.emplace(request); + return request; + } + + template <typename TEvResponse> + static void Complete(std::weak_ptr<TRequest> request, const typename TEvResponse::TResult& result) { + if (auto r = request.lock()) { + r->Complete(new TEvResponse(result)); + } + } + + template <typename TEvResponse> + static auto CreateCallback(std::weak_ptr<TRequest> request) { + return [request](const typename TEvResponse::TAsyncResult& result) { + Complete<TEvResponse>(request, result.GetValueSync()); + }; + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvPrivate::TEvComplete, Handle); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + THashSet<TRequestPtr, TRequestPtrHash> Requests; + +}; // TBaseProxyActor + +class TYdbProxy: public TBaseProxyActor<TYdbProxy> { + template <typename TEvResponse, typename TClient, typename... Args> + using TFunc = typename TEvResponse::TAsyncResult(TClient::*)(Args...); + + template <typename TClient> + TClient* EnsureClient(THolder<TClient>& client) { + if (!client) { + Y_VERIFY(AppData()->YdbDriver); + client.Reset(new TClient(*AppData()->YdbDriver, Settings)); + } + + return client.Get(); + } + + template <typename TClient> + TClient* EnsureClient() { + if constexpr (std::is_same_v<TClient, TSchemeClient>) { + return EnsureClient<TClient>(SchemeClient); + } else if constexpr (std::is_same_v<TClient, TTableClient>) { + return EnsureClient<TClient>(TableClient); + } else { + Y_FAIL("unreachable"); + } + } + + template <typename TEvResponse, typename TEvRequestPtr, typename TClient, typename... Args> + void Call(TEvRequestPtr& ev, TFunc<TEvResponse, TClient, Args...> func) { + auto* client = EnsureClient<TClient>(); + auto request = MakeRequest(ev->Sender, ev->Cookie); + auto args = std::move(ev->Get()->GetArgs()); + auto cb = CreateCallback<TEvResponse>(request); + + std::apply(func, std::tuple_cat(std::tie(client), std::move(args))).Subscribe(std::move(cb)); + } + + template <typename TEvResponse, typename TEvRequestPtr, typename TSession, typename... Args> + void CallSession(TEvRequestPtr& ev, TFunc<TEvResponse, TSession, Args...> func) { + auto* client = EnsureClient<TTableClient>(); + auto request = MakeRequest(ev->Sender, ev->Cookie); + auto args = std::move(ev->Get()->GetArgs()); + auto cb = [request, func, args = std::move(args)](const TAsyncCreateSessionResult& result) { + auto sessionResult = result.GetValueSync(); + if (!sessionResult.IsSuccess()) { + return Complete<TEvYdbProxy::TEvCreateSessionResponse>(request, sessionResult); + } + + auto session = sessionResult.GetSession(); + auto cb = CreateCallback<TEvResponse>(request); + std::apply(func, std::tuple_cat(std::tie(session), std::move(args))).Subscribe(std::move(cb)); + }; + + client->GetSession().Subscribe(std::move(cb)); + } + + void Handle(TEvYdbProxy::TEvMakeDirectoryRequest::TPtr& ev) { + Call<TEvYdbProxy::TEvMakeDirectoryResponse>(ev, &TSchemeClient::MakeDirectory); + } + + void Handle(TEvYdbProxy::TEvRemoveDirectoryRequest::TPtr& ev) { + Call<TEvYdbProxy::TEvRemoveDirectoryResponse>(ev, &TSchemeClient::RemoveDirectory); + } + + void Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { + Call<TEvYdbProxy::TEvDescribePathResponse>(ev, &TSchemeClient::DescribePath); + } + + void Handle(TEvYdbProxy::TEvListDirectoryRequest::TPtr& ev) { + Call<TEvYdbProxy::TEvListDirectoryResponse>(ev, &TSchemeClient::ListDirectory); + } + + void Handle(TEvYdbProxy::TEvModifyPermissionsRequest::TPtr& ev) { + Call<TEvYdbProxy::TEvModifyPermissionsResponse>(ev, &TSchemeClient::ModifyPermissions); + } + + void Handle(TEvYdbProxy::TEvCreateTableRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvCreateTableResponse>(ev, &TSession::CreateTable); + } + + void Handle(TEvYdbProxy::TEvDropTableRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvDropTableResponse>(ev, &TSession::DropTable); + } + + void Handle(TEvYdbProxy::TEvAlterTableRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvAlterTableResponse>(ev, &TSession::AlterTable); + } + + void Handle(TEvYdbProxy::TEvCopyTableRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvCopyTableResponse>(ev, &TSession::CopyTable); + } + + void Handle(TEvYdbProxy::TEvCopyTablesRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvCopyTablesResponse>(ev, &TSession::CopyTables); + } + + void Handle(TEvYdbProxy::TEvRenameTablesRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvRenameTablesResponse>(ev, &TSession::RenameTables); + } + + void Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { + CallSession<TEvYdbProxy::TEvDescribeTableResponse>(ev, &TSession::DescribeTable); + } + + static TClientSettings MakeSettings(const TString& endpoint, const TString& database) { + return TClientSettings() + .DiscoveryEndpoint(endpoint) + .DiscoveryMode(EDiscoveryMode::Async) + .Database(database); + } + + static TClientSettings MakeSettings(const TString& endpoint, const TString& database, const TString& token) { + return MakeSettings(endpoint, database) + .AuthToken(token); + } + + static TClientSettings MakeSettings(const TString& endpoint, const TString& database, const TStaticCredentials& credentials) { + return MakeSettings(endpoint, database) + .CredentialsProviderFactory(CreateLoginCredentialsProviderFactory({ + .User = credentials.GetUser(), + .Password = credentials.GetPassword(), + })); + } + +public: + template <typename... Args> + explicit TYdbProxy(Args&&... args) + : TBaseProxyActor(&TThis::StateWork) + , Settings(MakeSettings(std::forward<Args>(args)...)) + { + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + // Scheme + hFunc(TEvYdbProxy::TEvMakeDirectoryRequest, Handle); + hFunc(TEvYdbProxy::TEvRemoveDirectoryRequest, Handle); + hFunc(TEvYdbProxy::TEvDescribePathRequest, Handle); + hFunc(TEvYdbProxy::TEvListDirectoryRequest, Handle); + hFunc(TEvYdbProxy::TEvModifyPermissionsRequest, Handle); + // Table + hFunc(TEvYdbProxy::TEvCreateTableRequest, Handle); + hFunc(TEvYdbProxy::TEvDropTableRequest, Handle); + hFunc(TEvYdbProxy::TEvAlterTableRequest, Handle); + hFunc(TEvYdbProxy::TEvCopyTableRequest, Handle); + hFunc(TEvYdbProxy::TEvCopyTablesRequest, Handle); + hFunc(TEvYdbProxy::TEvRenameTablesRequest, Handle); + hFunc(TEvYdbProxy::TEvDescribeTableRequest, Handle); + + default: + return StateBase(ev, TlsActivationContext->AsActorContext()); + } + } + +private: + const TClientSettings Settings; + THolder<TSchemeClient> SchemeClient; + THolder<TTableClient> TableClient; + +}; // TYdbProxy + +IActor* CreateYdbProxy(const TString& endpoint, const TString& database) { + return new TYdbProxy(endpoint, database); +} + +IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TString& token) { + return new TYdbProxy(endpoint, database, token); +} + +IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TStaticCredentials& credentials) { + return new TYdbProxy(endpoint, database, credentials); +} + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h index 1011253856b..73590bfe8b6 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.h @@ -1,131 +1,131 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <ydb/core/base/defs.h> -#include <ydb/core/base/events.h> - -namespace NKikimrReplication { - class TStaticCredentials; -} - -namespace NKikimr { -namespace NReplication { - -#pragma push_macro("RemoveDirectory") -#undef RemoveDirectory - -struct TEvYdbProxy { - #define EV_REQUEST_RESPONSE(name) \ - Ev##name##Request, \ - Ev##name##Response - - enum EEv { - EvBegin = EventSpaceBegin(TKikimrEvents::ES_YDB_PROXY), - - EvScheme, - EV_REQUEST_RESPONSE(MakeDirectory), - EV_REQUEST_RESPONSE(RemoveDirectory), - EV_REQUEST_RESPONSE(DescribePath), - EV_REQUEST_RESPONSE(ListDirectory), - EV_REQUEST_RESPONSE(ModifyPermissions), - - EvTable = EvBegin + 1 * 100, - EV_REQUEST_RESPONSE(CreateSession), - EV_REQUEST_RESPONSE(CreateTable), - EV_REQUEST_RESPONSE(DropTable), - EV_REQUEST_RESPONSE(AlterTable), - EV_REQUEST_RESPONSE(CopyTable), - EV_REQUEST_RESPONSE(CopyTables), - EV_REQUEST_RESPONSE(RenameTables), - EV_REQUEST_RESPONSE(DescribeTable), - - EvEnd, - }; - - #undef EV_REQUEST_RESPONSE - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_YDB_PROXY), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_YDB_PROXY)"); - - template <typename TDerived, ui32 EventType, typename... Args> - struct TGenericRequest: public std::tuple<Args...>, public TEventLocal<TDerived, EventType> { - using TTupleArgs = std::tuple<Args...>; - - explicit TGenericRequest(Args... args) - : std::tuple<Args...>(args...) - { - } - - const TTupleArgs& GetArgs() const { - return *this; - } - - TTupleArgs&& GetArgs() { - return std::move(*this); - } - - using TBase = TGenericRequest<TDerived, EventType, Args...>; - }; - - template <typename TDerived, ui32 EventType, typename T> - struct TGenericResponse: public TEventLocal<TDerived, EventType> { - using TResult = T; - using TAsyncResult = NThreading::TFuture<TResult>; - - TResult Result; - - template <typename... Args> - explicit TGenericResponse(Args&&... args) - : Result(std::forward<Args>(args)...) - { - } - - using TBase = TGenericResponse<TDerived, EventType, T>; - }; - - #define DEFINE_GENERIC_REQUEST(name, ...) \ - struct TEv##name##Request: public TGenericRequest<TEv##name##Request, Ev##name##Request, __VA_ARGS__> { \ - using TBase::TBase; \ - } - - #define DEFINE_GENERIC_RESPONSE(name, result_t) \ - struct TEv##name##Response: public TGenericResponse<TEv##name##Response, Ev##name##Response, result_t> { \ - using TBase::TBase; \ - } - - #define DEFINE_GENERIC_REQUEST_RESPONSE(name, result_t, ...) \ - DEFINE_GENERIC_REQUEST(name, __VA_ARGS__); \ - DEFINE_GENERIC_RESPONSE(name, result_t) - - // Scheme - DEFINE_GENERIC_REQUEST_RESPONSE(MakeDirectory, NYdb::TStatus, TString, NYdb::NScheme::TMakeDirectorySettings); - DEFINE_GENERIC_REQUEST_RESPONSE(RemoveDirectory, NYdb::TStatus, TString, NYdb::NScheme::TRemoveDirectorySettings); - DEFINE_GENERIC_REQUEST_RESPONSE(DescribePath, NYdb::NScheme::TDescribePathResult, TString, NYdb::NScheme::TDescribePathSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(ListDirectory, NYdb::NScheme::TListDirectoryResult, TString, NYdb::NScheme::TListDirectorySettings); - DEFINE_GENERIC_REQUEST_RESPONSE(ModifyPermissions, NYdb::TStatus, TString, NYdb::NScheme::TModifyPermissionsSettings); - // Table - DEFINE_GENERIC_RESPONSE(CreateSession, NYdb::NTable::TCreateSessionResult); - DEFINE_GENERIC_REQUEST_RESPONSE(CreateTable, NYdb::TStatus, TString, NYdb::NTable::TTableDescription, NYdb::NTable::TCreateTableSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(DropTable, NYdb::TStatus, TString, NYdb::NTable::TDropTableSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(AlterTable, NYdb::TStatus, TString, NYdb::NTable::TAlterTableSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(CopyTable, NYdb::TStatus, TString, TString, NYdb::NTable::TCopyTableSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(CopyTables, NYdb::TStatus, TVector<NYdb::NTable::TCopyItem>, NYdb::NTable::TCopyTablesSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(RenameTables, NYdb::TStatus, TVector<NYdb::NTable::TRenameItem>, NYdb::NTable::TRenameTablesSettings); - DEFINE_GENERIC_REQUEST_RESPONSE(DescribeTable, NYdb::NTable::TDescribeTableResult, TString, NYdb::NTable::TDescribeTableSettings); - - #undef DEFINE_GENERIC_REQUEST_RESPONSE - #undef DEFINE_GENERIC_RESPONSE - #undef DEFINE_GENERIC_REQUEST - -}; // TEvYdbProxy - -#pragma pop_macro("RemoveDirectory") - -IActor* CreateYdbProxy(const TString& endpoint, const TString& database); -IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TString& token); -IActor* CreateYdbProxy(const TString& endpoint, const TString& database, - const NKikimrReplication::TStaticCredentials& credentials); - -} // NReplication -} // NKikimr + +#include <ydb/core/base/defs.h> +#include <ydb/core/base/events.h> + +namespace NKikimrReplication { + class TStaticCredentials; +} + +namespace NKikimr { +namespace NReplication { + +#pragma push_macro("RemoveDirectory") +#undef RemoveDirectory + +struct TEvYdbProxy { + #define EV_REQUEST_RESPONSE(name) \ + Ev##name##Request, \ + Ev##name##Response + + enum EEv { + EvBegin = EventSpaceBegin(TKikimrEvents::ES_YDB_PROXY), + + EvScheme, + EV_REQUEST_RESPONSE(MakeDirectory), + EV_REQUEST_RESPONSE(RemoveDirectory), + EV_REQUEST_RESPONSE(DescribePath), + EV_REQUEST_RESPONSE(ListDirectory), + EV_REQUEST_RESPONSE(ModifyPermissions), + + EvTable = EvBegin + 1 * 100, + EV_REQUEST_RESPONSE(CreateSession), + EV_REQUEST_RESPONSE(CreateTable), + EV_REQUEST_RESPONSE(DropTable), + EV_REQUEST_RESPONSE(AlterTable), + EV_REQUEST_RESPONSE(CopyTable), + EV_REQUEST_RESPONSE(CopyTables), + EV_REQUEST_RESPONSE(RenameTables), + EV_REQUEST_RESPONSE(DescribeTable), + + EvEnd, + }; + + #undef EV_REQUEST_RESPONSE + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_YDB_PROXY), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_YDB_PROXY)"); + + template <typename TDerived, ui32 EventType, typename... Args> + struct TGenericRequest: public std::tuple<Args...>, public TEventLocal<TDerived, EventType> { + using TTupleArgs = std::tuple<Args...>; + + explicit TGenericRequest(Args... args) + : std::tuple<Args...>(args...) + { + } + + const TTupleArgs& GetArgs() const { + return *this; + } + + TTupleArgs&& GetArgs() { + return std::move(*this); + } + + using TBase = TGenericRequest<TDerived, EventType, Args...>; + }; + + template <typename TDerived, ui32 EventType, typename T> + struct TGenericResponse: public TEventLocal<TDerived, EventType> { + using TResult = T; + using TAsyncResult = NThreading::TFuture<TResult>; + + TResult Result; + + template <typename... Args> + explicit TGenericResponse(Args&&... args) + : Result(std::forward<Args>(args)...) + { + } + + using TBase = TGenericResponse<TDerived, EventType, T>; + }; + + #define DEFINE_GENERIC_REQUEST(name, ...) \ + struct TEv##name##Request: public TGenericRequest<TEv##name##Request, Ev##name##Request, __VA_ARGS__> { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_RESPONSE(name, result_t) \ + struct TEv##name##Response: public TGenericResponse<TEv##name##Response, Ev##name##Response, result_t> { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_REQUEST_RESPONSE(name, result_t, ...) \ + DEFINE_GENERIC_REQUEST(name, __VA_ARGS__); \ + DEFINE_GENERIC_RESPONSE(name, result_t) + + // Scheme + DEFINE_GENERIC_REQUEST_RESPONSE(MakeDirectory, NYdb::TStatus, TString, NYdb::NScheme::TMakeDirectorySettings); + DEFINE_GENERIC_REQUEST_RESPONSE(RemoveDirectory, NYdb::TStatus, TString, NYdb::NScheme::TRemoveDirectorySettings); + DEFINE_GENERIC_REQUEST_RESPONSE(DescribePath, NYdb::NScheme::TDescribePathResult, TString, NYdb::NScheme::TDescribePathSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(ListDirectory, NYdb::NScheme::TListDirectoryResult, TString, NYdb::NScheme::TListDirectorySettings); + DEFINE_GENERIC_REQUEST_RESPONSE(ModifyPermissions, NYdb::TStatus, TString, NYdb::NScheme::TModifyPermissionsSettings); + // Table + DEFINE_GENERIC_RESPONSE(CreateSession, NYdb::NTable::TCreateSessionResult); + DEFINE_GENERIC_REQUEST_RESPONSE(CreateTable, NYdb::TStatus, TString, NYdb::NTable::TTableDescription, NYdb::NTable::TCreateTableSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(DropTable, NYdb::TStatus, TString, NYdb::NTable::TDropTableSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(AlterTable, NYdb::TStatus, TString, NYdb::NTable::TAlterTableSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(CopyTable, NYdb::TStatus, TString, TString, NYdb::NTable::TCopyTableSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(CopyTables, NYdb::TStatus, TVector<NYdb::NTable::TCopyItem>, NYdb::NTable::TCopyTablesSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(RenameTables, NYdb::TStatus, TVector<NYdb::NTable::TRenameItem>, NYdb::NTable::TRenameTablesSettings); + DEFINE_GENERIC_REQUEST_RESPONSE(DescribeTable, NYdb::NTable::TDescribeTableResult, TString, NYdb::NTable::TDescribeTableSettings); + + #undef DEFINE_GENERIC_REQUEST_RESPONSE + #undef DEFINE_GENERIC_RESPONSE + #undef DEFINE_GENERIC_REQUEST + +}; // TEvYdbProxy + +#pragma pop_macro("RemoveDirectory") + +IActor* CreateYdbProxy(const TString& endpoint, const TString& database); +IActor* CreateYdbProxy(const TString& endpoint, const TString& database, const TString& token); +IActor* CreateYdbProxy(const TString& endpoint, const TString& database, + const NKikimrReplication::TStaticCredentials& credentials); + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp index 34a599463b1..d536a6db195 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp @@ -1,471 +1,471 @@ -#include "ydb_proxy.h" - +#include "ydb_proxy.h" + #include <ydb/core/protos/replication.pb.h> #include <ydb/core/testlib/test_client.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <ydb/core/base/ticket_parser.h> - -namespace NKikimr { -namespace NReplication { - -Y_UNIT_TEST_SUITE(YdbProxyTests) { - template <bool UseDatabase = true> - class TEnv { - static constexpr char DomainName[] = "Root"; - - template <typename... Args> - void Init(Args&&... args) { - auto grpcPort = PortManager.GetPort(); - - Server.EnableGRpc(grpcPort); - Server.SetupDefaultProfiles(); - Client.InitRootScheme(DomainName); - - const auto endpoint = "localhost:" + ToString(grpcPort); - const auto database = "/" + ToString(DomainName); - - YdbProxy = Server.GetRuntime()->Register(CreateYdbProxy( - endpoint, UseDatabase ? database : "", std::forward<Args>(args)...)); - Sender = Server.GetRuntime()->AllocateEdgeActor(); - } - - void Login(ui64 schemeShardId, const TString& user, const TString& password) { - auto req = MakeHolder<NSchemeShard::TEvSchemeShard::TEvLogin>(); - req->Record.SetUser(user); - req->Record.SetPassword(password); + +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/core/base/ticket_parser.h> + +namespace NKikimr { +namespace NReplication { + +Y_UNIT_TEST_SUITE(YdbProxyTests) { + template <bool UseDatabase = true> + class TEnv { + static constexpr char DomainName[] = "Root"; + + template <typename... Args> + void Init(Args&&... args) { + auto grpcPort = PortManager.GetPort(); + + Server.EnableGRpc(grpcPort); + Server.SetupDefaultProfiles(); + Client.InitRootScheme(DomainName); + + const auto endpoint = "localhost:" + ToString(grpcPort); + const auto database = "/" + ToString(DomainName); + + YdbProxy = Server.GetRuntime()->Register(CreateYdbProxy( + endpoint, UseDatabase ? database : "", std::forward<Args>(args)...)); + Sender = Server.GetRuntime()->AllocateEdgeActor(); + } + + void Login(ui64 schemeShardId, const TString& user, const TString& password) { + auto req = MakeHolder<NSchemeShard::TEvSchemeShard::TEvLogin>(); + req->Record.SetUser(user); + req->Record.SetPassword(password); ForwardToTablet(*Server.GetRuntime(), schemeShardId, Sender, req.Release()); - - auto resp = Server.GetRuntime()->GrabEdgeEvent<NSchemeShard::TEvSchemeShard::TEvLoginResult>(Sender); - UNIT_ASSERT(resp->Get()->Record.GetError().empty()); - UNIT_ASSERT(!resp->Get()->Record.GetToken().empty()); - } - - public: - TEnv(bool init = true) - : Settings(Tests::TServerSettings(PortManager.GetPort()) - .SetDomainName(DomainName) - ) - , Server(Settings) - , Client(Settings) - { - if (init) { - Init(); - } - } - - explicit TEnv(const TString& user, const TString& password) - : TEnv(false) - { - NKikimrReplication::TStaticCredentials staticCreds; - staticCreds.SetUser(user); - staticCreds.SetPassword(password); - Init(staticCreds); - - const auto db = "/" + ToString(DomainName); - // create user & set owner - { - auto st = Client.CreateUser(db, user, password); - UNIT_ASSERT_VALUES_EQUAL(st, NMsgBusProxy::EResponseStatus::MSTATUS_OK); - - Client.ModifyOwner("/", DomainName, user); - } - // init security state - { - auto resp = Client.Ls(db); - - const auto& desc = resp->Record; - UNIT_ASSERT(desc.HasPathDescription()); - UNIT_ASSERT(desc.GetPathDescription().HasDomainDescription()); - UNIT_ASSERT(desc.GetPathDescription().GetDomainDescription().HasDomainKey()); - - Login(desc.GetPathDescription().GetDomainDescription().GetDomainKey().GetSchemeShard(), user, password); - } - // update security state - { - auto resp = Client.Ls(db); - - const auto& desc = resp->Record; - UNIT_ASSERT(desc.HasPathDescription()); - UNIT_ASSERT(desc.GetPathDescription().HasDomainDescription()); - UNIT_ASSERT(desc.GetPathDescription().GetDomainDescription().HasSecurityState()); - - const auto& secState = desc.GetPathDescription().GetDomainDescription().GetSecurityState(); - Server.GetRuntime()->Send(new IEventHandle(MakeTicketParserID(), Sender, - new TEvTicketParser::TEvUpdateLoginSecurityState(secState))); - } - } - - template <typename TEvResponse> - auto Send(IEventBase* ev) { - Server.GetRuntime()->Send(new IEventHandle(YdbProxy, Sender, ev)); - return Server.GetRuntime()->GrabEdgeEvent<TEvResponse>(Sender); - } - - private: - TPortManager PortManager; - Tests::TServerSettings Settings; - Tests::TServer Server; - Tests::TClient Client; - TActorId YdbProxy; - TActorId Sender; - }; - - Y_UNIT_TEST(MakeDirectory) { - TEnv env; - // ok - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // fail - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::BAD_REQUEST); - } - } - - Y_UNIT_TEST(RemoveDirectory) { - TEnv env; - // make - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // ok - { - auto ev = env.Send<TEvYdbProxy::TEvRemoveDirectoryResponse>( - new TEvYdbProxy::TEvRemoveDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // fail - { - auto ev = env.Send<TEvYdbProxy::TEvRemoveDirectoryResponse>( - new TEvYdbProxy::TEvRemoveDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - } - - Y_UNIT_TEST(DescribePath) { - TEnv env; - // describe root - { - auto ev = env.Send<TEvYdbProxy::TEvDescribePathResponse>( - new TEvYdbProxy::TEvDescribePathRequest("/Root", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - - const auto& entry = ev->Get()->Result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Root"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, NYdb::NScheme::ESchemeEntryType::Directory); - } - // make dir - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // describe dir - { - auto ev = env.Send<TEvYdbProxy::TEvDescribePathResponse>( - new TEvYdbProxy::TEvDescribePathRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - - const auto& entry = ev->Get()->Result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(entry.Name, "dir"); - UNIT_ASSERT_VALUES_EQUAL(entry.Type, NYdb::NScheme::ESchemeEntryType::Directory); - } - } - - Y_UNIT_TEST(ListDirectory) { - TEnv env; - // describe empty root - { - auto ev = env.Send<TEvYdbProxy::TEvListDirectoryResponse>( - new TEvYdbProxy::TEvListDirectoryRequest("/Root", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - - const auto& self = ev->Get()->Result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(self.Name, "Root"); - UNIT_ASSERT_VALUES_EQUAL(self.Type, NYdb::NScheme::ESchemeEntryType::Directory); - - const auto& children = ev->Get()->Result.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(children[0].Name, ".sys"); - UNIT_ASSERT_VALUES_EQUAL(children[0].Type, NYdb::NScheme::ESchemeEntryType::Directory); - } - // make dir - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // describe non-empty root - { - auto ev = env.Send<TEvYdbProxy::TEvListDirectoryResponse>( - new TEvYdbProxy::TEvListDirectoryRequest("/Root", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - - const auto& self = ev->Get()->Result.GetEntry(); - UNIT_ASSERT_VALUES_EQUAL(self.Name, "Root"); - UNIT_ASSERT_VALUES_EQUAL(self.Type, NYdb::NScheme::ESchemeEntryType::Directory); - - const auto& children = ev->Get()->Result.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 2); - } - } - - Y_UNIT_TEST(StaticCreds) { - TEnv env("user1", "password1"); - // make dir - { - auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( - new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - } - - Y_UNIT_TEST(CreateTable) { - TEnv<false> env; - // invalid key - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Float) // cannot be key - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - // ok, created - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // ok, exists - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - } - - Y_UNIT_TEST(DropTable) { - TEnv<false> env; - // create - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // ok - { - auto ev = env.Send<TEvYdbProxy::TEvDropTableResponse>( - new TEvYdbProxy::TEvDropTableRequest("/Root/table", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // fail - { - auto ev = env.Send<TEvYdbProxy::TEvDropTableResponse>( - new TEvYdbProxy::TEvDropTableRequest("/Root/table", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - } - - Y_UNIT_TEST(AlterTable) { - TEnv<false> env; - // fail - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendDropColumns("extra"); - - auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( - new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - // create - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .AddNullableColumn("extra", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // ok - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendDropColumns("extra"); - - auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( - new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // invalid column - { - auto settings = NYdb::NTable::TAlterTableSettings() - .AppendDropColumns("extra"); // not exist - - auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( - new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); - } - } - - Y_UNIT_TEST(CopyTable) { - TEnv<false> env; - // fail - { - auto ev = env.Send<TEvYdbProxy::TEvCopyTableResponse>( - new TEvYdbProxy::TEvCopyTableRequest("/Root/table", "/Root/copy", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); - } - // create - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // ok - { - auto ev = env.Send<TEvYdbProxy::TEvCopyTableResponse>( - new TEvYdbProxy::TEvCopyTableRequest("/Root/table", "/Root/copy", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - } - - Y_UNIT_TEST(CopyTables) { - TEnv<false> env; - - TVector<NYdb::NTable::TCopyItem> items{ - {"/Root/table1", "/Root/copy1"}, - {"/Root/table2", "/Root/copy2"}, - }; - - // create - for (const auto& item : items) { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest(item.SourcePath(), std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // copy - { - - auto ev = env.Send<TEvYdbProxy::TEvCopyTablesResponse>( - new TEvYdbProxy::TEvCopyTablesRequest(std::move(items), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - } - - Y_UNIT_TEST(DescribeTable) { - TEnv<false> env; - // create - { - auto schema = NYdb::NTable::TTableBuilder() - .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) - .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .Build(); - - auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( - new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - } - // describe - { - auto ev = env.Send<TEvYdbProxy::TEvDescribeTableResponse>( - new TEvYdbProxy::TEvDescribeTableRequest("/Root/table", {})); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()->Result.IsSuccess()); - - const auto& schema = ev->Get()->Result.GetTableDescription(); - // same pk - UNIT_ASSERT_EQUAL(schema.GetPrimaryKeyColumns().size(), 1); - UNIT_ASSERT_EQUAL(schema.GetPrimaryKeyColumns().at(0), "key"); - // same columns - UNIT_ASSERT_EQUAL(schema.GetColumns().size(), 2); - UNIT_ASSERT_EQUAL(schema.GetColumns().at(0).Name, "key"); - UNIT_ASSERT_EQUAL(schema.GetColumns().at(1).Name, "value"); - } - } - -} // YdbProxyTests - -} // NReplication -} // NKikimr + + auto resp = Server.GetRuntime()->GrabEdgeEvent<NSchemeShard::TEvSchemeShard::TEvLoginResult>(Sender); + UNIT_ASSERT(resp->Get()->Record.GetError().empty()); + UNIT_ASSERT(!resp->Get()->Record.GetToken().empty()); + } + + public: + TEnv(bool init = true) + : Settings(Tests::TServerSettings(PortManager.GetPort()) + .SetDomainName(DomainName) + ) + , Server(Settings) + , Client(Settings) + { + if (init) { + Init(); + } + } + + explicit TEnv(const TString& user, const TString& password) + : TEnv(false) + { + NKikimrReplication::TStaticCredentials staticCreds; + staticCreds.SetUser(user); + staticCreds.SetPassword(password); + Init(staticCreds); + + const auto db = "/" + ToString(DomainName); + // create user & set owner + { + auto st = Client.CreateUser(db, user, password); + UNIT_ASSERT_VALUES_EQUAL(st, NMsgBusProxy::EResponseStatus::MSTATUS_OK); + + Client.ModifyOwner("/", DomainName, user); + } + // init security state + { + auto resp = Client.Ls(db); + + const auto& desc = resp->Record; + UNIT_ASSERT(desc.HasPathDescription()); + UNIT_ASSERT(desc.GetPathDescription().HasDomainDescription()); + UNIT_ASSERT(desc.GetPathDescription().GetDomainDescription().HasDomainKey()); + + Login(desc.GetPathDescription().GetDomainDescription().GetDomainKey().GetSchemeShard(), user, password); + } + // update security state + { + auto resp = Client.Ls(db); + + const auto& desc = resp->Record; + UNIT_ASSERT(desc.HasPathDescription()); + UNIT_ASSERT(desc.GetPathDescription().HasDomainDescription()); + UNIT_ASSERT(desc.GetPathDescription().GetDomainDescription().HasSecurityState()); + + const auto& secState = desc.GetPathDescription().GetDomainDescription().GetSecurityState(); + Server.GetRuntime()->Send(new IEventHandle(MakeTicketParserID(), Sender, + new TEvTicketParser::TEvUpdateLoginSecurityState(secState))); + } + } + + template <typename TEvResponse> + auto Send(IEventBase* ev) { + Server.GetRuntime()->Send(new IEventHandle(YdbProxy, Sender, ev)); + return Server.GetRuntime()->GrabEdgeEvent<TEvResponse>(Sender); + } + + private: + TPortManager PortManager; + Tests::TServerSettings Settings; + Tests::TServer Server; + Tests::TClient Client; + TActorId YdbProxy; + TActorId Sender; + }; + + Y_UNIT_TEST(MakeDirectory) { + TEnv env; + // ok + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // fail + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::BAD_REQUEST); + } + } + + Y_UNIT_TEST(RemoveDirectory) { + TEnv env; + // make + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // ok + { + auto ev = env.Send<TEvYdbProxy::TEvRemoveDirectoryResponse>( + new TEvYdbProxy::TEvRemoveDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // fail + { + auto ev = env.Send<TEvYdbProxy::TEvRemoveDirectoryResponse>( + new TEvYdbProxy::TEvRemoveDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + } + } + + Y_UNIT_TEST(DescribePath) { + TEnv env; + // describe root + { + auto ev = env.Send<TEvYdbProxy::TEvDescribePathResponse>( + new TEvYdbProxy::TEvDescribePathRequest("/Root", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + + const auto& entry = ev->Get()->Result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Root"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, NYdb::NScheme::ESchemeEntryType::Directory); + } + // make dir + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // describe dir + { + auto ev = env.Send<TEvYdbProxy::TEvDescribePathResponse>( + new TEvYdbProxy::TEvDescribePathRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + + const auto& entry = ev->Get()->Result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.Name, "dir"); + UNIT_ASSERT_VALUES_EQUAL(entry.Type, NYdb::NScheme::ESchemeEntryType::Directory); + } + } + + Y_UNIT_TEST(ListDirectory) { + TEnv env; + // describe empty root + { + auto ev = env.Send<TEvYdbProxy::TEvListDirectoryResponse>( + new TEvYdbProxy::TEvListDirectoryRequest("/Root", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + + const auto& self = ev->Get()->Result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(self.Name, "Root"); + UNIT_ASSERT_VALUES_EQUAL(self.Type, NYdb::NScheme::ESchemeEntryType::Directory); + + const auto& children = ev->Get()->Result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(children[0].Name, ".sys"); + UNIT_ASSERT_VALUES_EQUAL(children[0].Type, NYdb::NScheme::ESchemeEntryType::Directory); + } + // make dir + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // describe non-empty root + { + auto ev = env.Send<TEvYdbProxy::TEvListDirectoryResponse>( + new TEvYdbProxy::TEvListDirectoryRequest("/Root", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + + const auto& self = ev->Get()->Result.GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(self.Name, "Root"); + UNIT_ASSERT_VALUES_EQUAL(self.Type, NYdb::NScheme::ESchemeEntryType::Directory); + + const auto& children = ev->Get()->Result.GetChildren(); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 2); + } + } + + Y_UNIT_TEST(StaticCreds) { + TEnv env("user1", "password1"); + // make dir + { + auto ev = env.Send<TEvYdbProxy::TEvMakeDirectoryResponse>( + new TEvYdbProxy::TEvMakeDirectoryRequest("/Root/dir", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + } + + Y_UNIT_TEST(CreateTable) { + TEnv<false> env; + // invalid key + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Float) // cannot be key + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + } + // ok, created + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // ok, exists + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + } + + Y_UNIT_TEST(DropTable) { + TEnv<false> env; + // create + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // ok + { + auto ev = env.Send<TEvYdbProxy::TEvDropTableResponse>( + new TEvYdbProxy::TEvDropTableRequest("/Root/table", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // fail + { + auto ev = env.Send<TEvYdbProxy::TEvDropTableResponse>( + new TEvYdbProxy::TEvDropTableRequest("/Root/table", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + } + } + + Y_UNIT_TEST(AlterTable) { + TEnv<false> env; + // fail + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendDropColumns("extra"); + + auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( + new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + } + // create + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .AddNullableColumn("extra", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // ok + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendDropColumns("extra"); + + auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( + new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // invalid column + { + auto settings = NYdb::NTable::TAlterTableSettings() + .AppendDropColumns("extra"); // not exist + + auto ev = env.Send<TEvYdbProxy::TEvAlterTableResponse>( + new TEvYdbProxy::TEvAlterTableRequest("/Root/table", settings)); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::GENERIC_ERROR); + } + } + + Y_UNIT_TEST(CopyTable) { + TEnv<false> env; + // fail + { + auto ev = env.Send<TEvYdbProxy::TEvCopyTableResponse>( + new TEvYdbProxy::TEvCopyTableRequest("/Root/table", "/Root/copy", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(!ev->Get()->Result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); + } + // create + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // ok + { + auto ev = env.Send<TEvYdbProxy::TEvCopyTableResponse>( + new TEvYdbProxy::TEvCopyTableRequest("/Root/table", "/Root/copy", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + } + + Y_UNIT_TEST(CopyTables) { + TEnv<false> env; + + TVector<NYdb::NTable::TCopyItem> items{ + {"/Root/table1", "/Root/copy1"}, + {"/Root/table2", "/Root/copy2"}, + }; + + // create + for (const auto& item : items) { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest(item.SourcePath(), std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // copy + { + + auto ev = env.Send<TEvYdbProxy::TEvCopyTablesResponse>( + new TEvYdbProxy::TEvCopyTablesRequest(std::move(items), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + } + + Y_UNIT_TEST(DescribeTable) { + TEnv<false> env; + // create + { + auto schema = NYdb::NTable::TTableBuilder() + .AddNullableColumn("key", NYdb::EPrimitiveType::Uint64) + .AddNullableColumn("value", NYdb::EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + + auto ev = env.Send<TEvYdbProxy::TEvCreateTableResponse>( + new TEvYdbProxy::TEvCreateTableRequest("/Root/table", std::move(schema), {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + // describe + { + auto ev = env.Send<TEvYdbProxy::TEvDescribeTableResponse>( + new TEvYdbProxy::TEvDescribeTableRequest("/Root/table", {})); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + + const auto& schema = ev->Get()->Result.GetTableDescription(); + // same pk + UNIT_ASSERT_EQUAL(schema.GetPrimaryKeyColumns().size(), 1); + UNIT_ASSERT_EQUAL(schema.GetPrimaryKeyColumns().at(0), "key"); + // same columns + UNIT_ASSERT_EQUAL(schema.GetColumns().size(), 2); + UNIT_ASSERT_EQUAL(schema.GetColumns().at(0).Name, "key"); + UNIT_ASSERT_EQUAL(schema.GetColumns().at(1).Name, "value"); + } + } + +} // YdbProxyTests + +} // NReplication +} // NKikimr diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 7a184a842b4..8e778a4a1d8 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -1,10 +1,10 @@ -#include "cache.h" -#include "double_indexed.h" -#include "events.h" -#include "helpers.h" -#include "monitorable_actor.h" -#include "subscriber.h" - +#include "cache.h" +#include "double_indexed.h" +#include "events.h" +#include "helpers.h" +#include "monitorable_actor.h" +#include "subscriber.h" + #include <ydb/core/tx/datashard/sys_tables.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> @@ -17,281 +17,281 @@ #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/tx/schemeshard/schemeshard_types.h> #include <ydb/core/util/yverify_stream.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> -#include <library/cpp/json/writer/json.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/variant.h> -#include <util/generic/vector.h> -#include <util/generic/xrange.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NSchemeBoard { - -#define SBC_LOG_T(stream) SB_LOG_T(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_D(stream) SB_LOG_D(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_N(stream) SB_LOG_N(TX_PROXY_SCHEME_CACHE, stream) +#include <library/cpp/json/writer/json.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/variant.h> +#include <util/generic/vector.h> +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NSchemeBoard { + +#define SBC_LOG_T(stream) SB_LOG_T(TX_PROXY_SCHEME_CACHE, stream) +#define SBC_LOG_D(stream) SB_LOG_D(TX_PROXY_SCHEME_CACHE, stream) +#define SBC_LOG_N(stream) SB_LOG_N(TX_PROXY_SCHEME_CACHE, stream) #define SBC_LOG_W(stream) SB_LOG_W(TX_PROXY_SCHEME_CACHE, stream) - -using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; -using TEvNavigateResult = TEvTxProxySchemeCache::TEvNavigateKeySetResult; -using TNavigate = NSchemeCache::TSchemeCacheNavigate; -using TNavigateContext = NSchemeCache::TSchemeCacheNavigateContext; -using TNavigateContextPtr = TIntrusivePtr<TNavigateContext>; - -using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; -using TEvResolveResult = TEvTxProxySchemeCache::TEvResolveKeySetResult; -using TResolve = NSchemeCache::TSchemeCacheRequest; -using TResolveContext = NSchemeCache::TSchemeCacheRequestContext; -using TResolveContextPtr = TIntrusivePtr<TResolveContext>; - + +using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; +using TEvNavigateResult = TEvTxProxySchemeCache::TEvNavigateKeySetResult; +using TNavigate = NSchemeCache::TSchemeCacheNavigate; +using TNavigateContext = NSchemeCache::TSchemeCacheNavigateContext; +using TNavigateContextPtr = TIntrusivePtr<TNavigateContext>; + +using TEvResolve = TEvTxProxySchemeCache::TEvResolveKeySet; +using TEvResolveResult = TEvTxProxySchemeCache::TEvResolveKeySetResult; +using TResolve = NSchemeCache::TSchemeCacheRequest; +using TResolveContext = NSchemeCache::TSchemeCacheRequestContext; +using TResolveContextPtr = TIntrusivePtr<TResolveContext>; + using TVariantContextPtr = std::variant<TNavigateContextPtr, TResolveContextPtr>; - -namespace { - - void SetError(TNavigateContext* context, TNavigate::TEntry& entry, TNavigate::EStatus status) { - ++context->Request->ErrorCount; - entry.Status = status; - } - - void SetError(TResolveContext* context, TResolve::TEntry& entry, TResolve::EStatus entryStatus, TKeyDesc::EStatus keyDescStatus) { - entry.Status = entryStatus; - entry.KeyDescription->Status = keyDescStatus; - ++context->Request->ErrorCount; - } - - void SetRootUnknown(TNavigateContext* context, TNavigate::TEntry& entry) { - SetError(context, entry, TNavigate::EStatus::RootUnknown); - } - - void SetRootUnknown(TResolveContext* context, TResolve::TEntry&) { - ++context->Request->ErrorCount; - } - - void SetPathNotExist(TNavigateContext* context, TNavigate::TEntry& entry) { - SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - - void SetPathNotExist(TResolveContext* context, TResolve::TEntry& entry) { - SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); - } - - void SetLookupError(TNavigateContext* context, TNavigate::TEntry& entry) { - SetError(context, entry, TNavigate::EStatus::LookupError); - } - - void SetLookupError(TResolveContext* context, TResolve::TEntry& entry) { - SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); - } - - template <typename TRequest, typename TEvRequest, typename TDerived> - class TDbResolver: public TActorBootstrapped<TDerived> { - void Handle() { - TlsActivationContext->Send(new IEventHandle(Cache, Sender, new TEvRequest(Request.Release()))); - this->PassAway(); - } - - public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SCHEME_BOARD_DB_RESOLVER; - } - + +namespace { + + void SetError(TNavigateContext* context, TNavigate::TEntry& entry, TNavigate::EStatus status) { + ++context->Request->ErrorCount; + entry.Status = status; + } + + void SetError(TResolveContext* context, TResolve::TEntry& entry, TResolve::EStatus entryStatus, TKeyDesc::EStatus keyDescStatus) { + entry.Status = entryStatus; + entry.KeyDescription->Status = keyDescStatus; + ++context->Request->ErrorCount; + } + + void SetRootUnknown(TNavigateContext* context, TNavigate::TEntry& entry) { + SetError(context, entry, TNavigate::EStatus::RootUnknown); + } + + void SetRootUnknown(TResolveContext* context, TResolve::TEntry&) { + ++context->Request->ErrorCount; + } + + void SetPathNotExist(TNavigateContext* context, TNavigate::TEntry& entry) { + SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + } + + void SetPathNotExist(TResolveContext* context, TResolve::TEntry& entry) { + SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); + } + + void SetLookupError(TNavigateContext* context, TNavigate::TEntry& entry) { + SetError(context, entry, TNavigate::EStatus::LookupError); + } + + void SetLookupError(TResolveContext* context, TResolve::TEntry& entry) { + SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); + } + + template <typename TRequest, typename TEvRequest, typename TDerived> + class TDbResolver: public TActorBootstrapped<TDerived> { + void Handle() { + TlsActivationContext->Send(new IEventHandle(Cache, Sender, new TEvRequest(Request.Release()))); + this->PassAway(); + } + + public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SCHEME_BOARD_DB_RESOLVER; + } + TDbResolver(const TActorId& cache, const TActorId& sender, THolder<TRequest> request, ui64 domainOwnerId) - : Cache(cache) - , Sender(sender) - , Request(std::move(request)) - , DomainOwnerId(domainOwnerId) - { - } - - void Bootstrap() { - TNavigate::TEntry entry; - entry.Path = SplitPath(Request->DatabaseName); - entry.Operation = TNavigate::EOp::OpPath; - entry.RedirectRequired = false; - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.emplace_back(std::move(entry)); - request->DomainOwnerId = DomainOwnerId; - - this->Send(Cache, new TEvNavigate(request.Release())); - this->Become(&TDerived::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - } - } - - using TBase = TDbResolver<TRequest, TEvRequest, TDerived>; - - private: + : Cache(cache) + , Sender(sender) + , Request(std::move(request)) + , DomainOwnerId(domainOwnerId) + { + } + + void Bootstrap() { + TNavigate::TEntry entry; + entry.Path = SplitPath(Request->DatabaseName); + entry.Operation = TNavigate::EOp::OpPath; + entry.RedirectRequired = false; + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.emplace_back(std::move(entry)); + request->DomainOwnerId = DomainOwnerId; + + this->Send(Cache, new TEvNavigate(request.Release())); + this->Become(&TDerived::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + sFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + } + } + + using TBase = TDbResolver<TRequest, TEvRequest, TDerived>; + + private: const TActorId Cache; const TActorId Sender; - THolder<TRequest> Request; - const ui64 DomainOwnerId; - - }; // TDbResolver - - class TDbResolverNavigate: public TDbResolver<TNavigate, TEvNavigate, TDbResolverNavigate> { - public: - using TBase::TBase; - }; - - class TDbResolverResolve: public TDbResolver<TResolve, TEvResolve, TDbResolverResolve> { - public: - using TBase::TBase; - }; - + THolder<TRequest> Request; + const ui64 DomainOwnerId; + + }; // TDbResolver + + class TDbResolverNavigate: public TDbResolver<TNavigate, TEvNavigate, TDbResolverNavigate> { + public: + using TBase::TBase; + }; + + class TDbResolverResolve: public TDbResolver<TResolve, TEvResolve, TDbResolverResolve> { + public: + using TBase::TBase; + }; + IActor* CreateDbResolver(const TActorId& cache, const TActorId& sender, THolder<TNavigate> request, ui64 domainOwnerId) { - return new TDbResolverNavigate(cache, sender, std::move(request), domainOwnerId); - } - + return new TDbResolverNavigate(cache, sender, std::move(request), domainOwnerId); + } + IActor* CreateDbResolver(const TActorId& cache, const TActorId& sender, THolder<TResolve> request, ui64 domainOwnerId) { - return new TDbResolverResolve(cache, sender, std::move(request), domainOwnerId); - } - - template <typename TContextPtr, typename TEvResult, typename TDerived> - class TAclChecker: public TActorBootstrapped<TDerived> { - static TIntrusivePtr<TSecurityObject> GetSecurityObject(const TNavigate::TEntry& entry) { - return entry.SecurityObject; - } - - static TIntrusivePtr<TSecurityObject> GetSecurityObject(const TResolve::TEntry& entry) { - return entry.KeyDescription->SecurityObject; - } - - static ui32 GetAccess(const TNavigate::TEntry&) { - return NACLib::EAccessRights::DescribeSchema; - } - - static ui32 GetAccess(const TResolve::TEntry& entry) { - return entry.Access; - } - - static void SetErrorAndClear(TNavigateContext* context, TNavigate::TEntry& entry) { - SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - - switch (entry.RequestType) { - case TNavigate::TEntry::ERequestType::ByPath: - entry.TableId = TTableId(); - break; - case TNavigate::TEntry::ERequestType::ByTableId: - entry.Path.clear(); - break; - } - + return new TDbResolverResolve(cache, sender, std::move(request), domainOwnerId); + } + + template <typename TContextPtr, typename TEvResult, typename TDerived> + class TAclChecker: public TActorBootstrapped<TDerived> { + static TIntrusivePtr<TSecurityObject> GetSecurityObject(const TNavigate::TEntry& entry) { + return entry.SecurityObject; + } + + static TIntrusivePtr<TSecurityObject> GetSecurityObject(const TResolve::TEntry& entry) { + return entry.KeyDescription->SecurityObject; + } + + static ui32 GetAccess(const TNavigate::TEntry&) { + return NACLib::EAccessRights::DescribeSchema; + } + + static ui32 GetAccess(const TResolve::TEntry& entry) { + return entry.Access; + } + + static void SetErrorAndClear(TNavigateContext* context, TNavigate::TEntry& entry) { + SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + + switch (entry.RequestType) { + case TNavigate::TEntry::ERequestType::ByPath: + entry.TableId = TTableId(); + break; + case TNavigate::TEntry::ERequestType::ByTableId: + entry.Path.clear(); + break; + } + entry.Self.Drop(); - entry.SecurityObject.Drop(); - entry.DomainInfo.Drop(); - entry.Kind = TNavigate::KindUnknown; - entry.Attributes.clear(); - entry.ListNodeEntry.Drop(); - entry.DomainDescription.Drop(); - entry.Columns.clear(); + entry.SecurityObject.Drop(); + entry.DomainInfo.Drop(); + entry.Kind = TNavigate::KindUnknown; + entry.Attributes.clear(); + entry.ListNodeEntry.Drop(); + entry.DomainDescription.Drop(); + entry.Columns.clear(); entry.NotNullColumns.clear(); - entry.Indexes.clear(); - entry.CdcStreams.clear(); - entry.RTMRVolumeInfo.Drop(); - entry.KesusInfo.Drop(); - entry.SolomonVolumeInfo.Drop(); - entry.PQGroupInfo.Drop(); + entry.Indexes.clear(); + entry.CdcStreams.clear(); + entry.RTMRVolumeInfo.Drop(); + entry.KesusInfo.Drop(); + entry.SolomonVolumeInfo.Drop(); + entry.PQGroupInfo.Drop(); entry.OlapStoreInfo.Drop(); entry.OlapTableInfo.Drop(); - entry.CdcStreamInfo.Drop(); + entry.CdcStreamInfo.Drop(); entry.SequenceInfo.Drop(); - entry.ReplicationInfo.Drop(); - } - - static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry) { - SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); - - entry.Kind = TResolve::KindUnknown; - entry.DomainInfo.Drop(); - TKeyDesc& keyDesc = *entry.KeyDescription; - keyDesc.ColumnInfos.clear(); - keyDesc.Partitions.clear(); - keyDesc.SecurityObject.Drop(); - } - - void SendResult() { - SBC_LOG_D("Send result" - << ": self# " << this->SelfId() - << ", recipient# " << Context->Sender - << ", result# " << Context->Request->ToString(*AppData()->TypeRegistry)); - - this->Send(Context->Sender, new TEvResult(Context->Request.Release())); - this->PassAway(); - } - - public: - explicit TAclChecker(TContextPtr context) - : Context(context) - { - Y_VERIFY(!Context->WaitCounter); - } - - void Bootstrap(const TActorContext&) { - if (Context->Request->UserToken == nullptr) { - SendResult(); - return; - } - - for (auto& entry : Context->Request->ResultSet) { - auto securityObject = GetSecurityObject(entry); - if (securityObject == nullptr) { - continue; - } - - const ui32 access = GetAccess(entry); - if (!securityObject->CheckAccess(access, *Context->Request->UserToken)) { - SBC_LOG_W("Access denied" - << ": self# " << this->SelfId() - << ", for# " << Context->Request->UserToken->GetUserSID() - << ", access# " << NACLib::AccessRightsToString(access)); - - SetErrorAndClear(Context.Get(), entry); - } - } - - SendResult(); - } - - using TBase = TAclChecker<TContextPtr, TEvResult, TDerived>; - - private: - TContextPtr Context; - - }; // TAclChecker - - class TAclCheckerNavigate: public TAclChecker<TNavigateContextPtr, TEvNavigateResult, TAclCheckerNavigate> { - public: - using TBase::TBase; - }; - - class TAclCheckerResolve: public TAclChecker<TResolveContextPtr, TEvResolveResult, TAclCheckerResolve> { - public: - using TBase::TBase; - }; - - IActor* CreateAclChecker(TNavigateContextPtr context) { - return new TAclCheckerNavigate(context); - } - - IActor* CreateAclChecker(TResolveContextPtr context) { - return new TAclCheckerResolve(context); - } - - class TWatchCache: public TMonitorableActor<TWatchCache> { + entry.ReplicationInfo.Drop(); + } + + static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry) { + SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); + + entry.Kind = TResolve::KindUnknown; + entry.DomainInfo.Drop(); + TKeyDesc& keyDesc = *entry.KeyDescription; + keyDesc.ColumnInfos.clear(); + keyDesc.Partitions.clear(); + keyDesc.SecurityObject.Drop(); + } + + void SendResult() { + SBC_LOG_D("Send result" + << ": self# " << this->SelfId() + << ", recipient# " << Context->Sender + << ", result# " << Context->Request->ToString(*AppData()->TypeRegistry)); + + this->Send(Context->Sender, new TEvResult(Context->Request.Release())); + this->PassAway(); + } + + public: + explicit TAclChecker(TContextPtr context) + : Context(context) + { + Y_VERIFY(!Context->WaitCounter); + } + + void Bootstrap(const TActorContext&) { + if (Context->Request->UserToken == nullptr) { + SendResult(); + return; + } + + for (auto& entry : Context->Request->ResultSet) { + auto securityObject = GetSecurityObject(entry); + if (securityObject == nullptr) { + continue; + } + + const ui32 access = GetAccess(entry); + if (!securityObject->CheckAccess(access, *Context->Request->UserToken)) { + SBC_LOG_W("Access denied" + << ": self# " << this->SelfId() + << ", for# " << Context->Request->UserToken->GetUserSID() + << ", access# " << NACLib::AccessRightsToString(access)); + + SetErrorAndClear(Context.Get(), entry); + } + } + + SendResult(); + } + + using TBase = TAclChecker<TContextPtr, TEvResult, TDerived>; + + private: + TContextPtr Context; + + }; // TAclChecker + + class TAclCheckerNavigate: public TAclChecker<TNavigateContextPtr, TEvNavigateResult, TAclCheckerNavigate> { + public: + using TBase::TBase; + }; + + class TAclCheckerResolve: public TAclChecker<TResolveContextPtr, TEvResolveResult, TAclCheckerResolve> { + public: + using TBase::TBase; + }; + + IActor* CreateAclChecker(TNavigateContextPtr context) { + return new TAclCheckerNavigate(context); + } + + IActor* CreateAclChecker(TResolveContextPtr context) { + return new TAclCheckerResolve(context); + } + + class TWatchCache: public TMonitorableActor<TWatchCache> { using TBase = TMonitorableActor<TWatchCache>; private: @@ -355,14 +355,14 @@ namespace { } private: - STATEFN(StateWork) { + STATEFN(StateWork) { switch (ev->GetTypeRewrite()) { hFunc(TEvTxProxySchemeCache::TEvWatchPathId, Handle); hFunc(TEvTxProxySchemeCache::TEvWatchRemove, Handle); hFunc(TSchemeBoardEvents::TEvNotifyUpdate, Handle); hFunc(TSchemeBoardEvents::TEvNotifyDelete, Handle); - - sFunc(TEvents::TEvPoison, PassAway); + + sFunc(TEvents::TEvPoison, PassAway); } } @@ -405,7 +405,7 @@ namespace { } if (msg->DescribeSchemeResult.HasStatus()) { - entry->Path = msg->DescribeSchemeResult.GetPath(); + entry->Path = msg->DescribeSchemeResult.GetPath(); entry->Result = NSchemeCache::TDescribeResult::Create(std::move(msg->DescribeSchemeResult)); entry->Deleted = false; entry->Unavailable = false; @@ -511,256 +511,256 @@ namespace { THashMap<TActorId, TEntry*> EntriesBySubscriber; TMultiMap<TWatcher, TEntry*, TWatcherCompare> EntriesByWatcher; - }; // TWatchCache - -} // anonymous - -class TSchemeCache: public TMonitorableActor<TSchemeCache> { - class TCounters { - using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; - using THistogramPtr = NMonitoring::THistogramPtr; - - TCounterPtr InFlight; - TCounterPtr Requests; - TCounterPtr Hits; - TCounterPtr Misses; - TCounterPtr Syncs; - THistogramPtr Latency; - - TCounterPtr PerEntryHits; - TCounterPtr PerEntryMisses; - TCounterPtr PerEntrySyncs; - - public: - explicit TCounters(NMonitoring::TDynamicCounterPtr counters) - : InFlight(counters->GetCounter("InFlight", false)) - , Requests(counters->GetCounter("Requests", true)) - , Hits(counters->GetCounter("Hits", true)) - , Misses(counters->GetCounter("Misses", true)) - , Syncs(counters->GetCounter("Syncs", true)) - , Latency(counters->GetHistogram("LatencyMs", NMonitoring::ExponentialHistogram(10, 4, 1))) - , PerEntryHits(counters->GetCounter("PerEntry/Hits", true)) - , PerEntryMisses(counters->GetCounter("PerEntry/Misses", true)) - , PerEntrySyncs(counters->GetCounter("PerEntry/Syncs", true)) - { - } - - void StartRequest(ui64 entryCount, ui64 waitCount, ui64 syncCount) { - *InFlight += 1; - *Requests += 1; - - if (!waitCount || waitCount == syncCount) { - *Hits += 1; - } else { - *Misses += 1; - } - if (syncCount) { - *Syncs += 1; - } - - *PerEntryHits += (entryCount - (waitCount - syncCount)); - *PerEntryMisses += (waitCount - syncCount); - *PerEntrySyncs += syncCount; - } - - void FinishRequest(const TDuration& latency) { - *InFlight -= 1; - Latency->Collect(latency.MilliSeconds()); - } - }; - - struct TSubscriber { - enum class EType: ui8 { - // from low to high priority - Empty, - ByPathId, - ByPath, - }; - + }; // TWatchCache + +} // anonymous + +class TSchemeCache: public TMonitorableActor<TSchemeCache> { + class TCounters { + using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; + using THistogramPtr = NMonitoring::THistogramPtr; + + TCounterPtr InFlight; + TCounterPtr Requests; + TCounterPtr Hits; + TCounterPtr Misses; + TCounterPtr Syncs; + THistogramPtr Latency; + + TCounterPtr PerEntryHits; + TCounterPtr PerEntryMisses; + TCounterPtr PerEntrySyncs; + + public: + explicit TCounters(NMonitoring::TDynamicCounterPtr counters) + : InFlight(counters->GetCounter("InFlight", false)) + , Requests(counters->GetCounter("Requests", true)) + , Hits(counters->GetCounter("Hits", true)) + , Misses(counters->GetCounter("Misses", true)) + , Syncs(counters->GetCounter("Syncs", true)) + , Latency(counters->GetHistogram("LatencyMs", NMonitoring::ExponentialHistogram(10, 4, 1))) + , PerEntryHits(counters->GetCounter("PerEntry/Hits", true)) + , PerEntryMisses(counters->GetCounter("PerEntry/Misses", true)) + , PerEntrySyncs(counters->GetCounter("PerEntry/Syncs", true)) + { + } + + void StartRequest(ui64 entryCount, ui64 waitCount, ui64 syncCount) { + *InFlight += 1; + *Requests += 1; + + if (!waitCount || waitCount == syncCount) { + *Hits += 1; + } else { + *Misses += 1; + } + if (syncCount) { + *Syncs += 1; + } + + *PerEntryHits += (entryCount - (waitCount - syncCount)); + *PerEntryMisses += (waitCount - syncCount); + *PerEntrySyncs += syncCount; + } + + void FinishRequest(const TDuration& latency) { + *InFlight -= 1; + Latency->Collect(latency.MilliSeconds()); + } + }; + + struct TSubscriber { + enum class EType: ui8 { + // from low to high priority + Empty, + ByPathId, + ByPath, + }; + TActorId Subscriber; - ui64 DomainOwnerId; - EType Type; - mutable ui64 SyncCookie; - - TSubscriber() - : DomainOwnerId(0) - , Type(EType::Empty) - , SyncCookie(0) - { - } - + ui64 DomainOwnerId; + EType Type; + mutable ui64 SyncCookie; + + TSubscriber() + : DomainOwnerId(0) + , Type(EType::Empty) + , SyncCookie(0) + { + } + explicit TSubscriber(const TActorId& subscriber, const ui64 domainOwnerId, const TPathId&) - : Subscriber(subscriber) - , DomainOwnerId(domainOwnerId) - , Type(EType::ByPathId) - , SyncCookie(0) - { - } - + : Subscriber(subscriber) + , DomainOwnerId(domainOwnerId) + , Type(EType::ByPathId) + , SyncCookie(0) + { + } + explicit TSubscriber(const TActorId& subscriber, const ui64 domainOwnerId, const TString&) - : Subscriber(subscriber) - , DomainOwnerId(domainOwnerId) - , Type(EType::ByPath) - , SyncCookie(0) - { - } - - TString ToString() const { - return TStringBuilder() << "{" - << " Subscriber: " << Subscriber - << " DomainOwnerId: " << DomainOwnerId - << " Type: " << static_cast<ui32>(Type) - << " SyncCookie: " << SyncCookie - << " }"; - } - - bool operator<(const TSubscriber& x) const { - return Type < x.Type; - } - - explicit operator bool() const { - return bool(Subscriber); - } - }; - - struct TResponseProps { - ui64 Cookie; - bool IsSync; - bool Partial; - - TResponseProps() - : Cookie(0) - , IsSync(false) - , Partial(false) - { - } - - explicit TResponseProps(ui64 cookie, bool isSync, bool partial) - : Cookie(cookie) - , IsSync(isSync) - , Partial(partial) - { - } - - static TResponseProps FromEvent(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { - return TResponseProps(ev->Cookie, false, false); - } - - static TResponseProps FromEvent(TSchemeBoardEvents::TEvNotifyDelete::TPtr& ev) { - return TResponseProps(ev->Cookie, false, false); - } - - static TResponseProps FromEvent(TSchemeBoardEvents::TEvSyncResponse::TPtr& ev) { - return TResponseProps(ev->Cookie, true, ev->Get()->Partial); - } - - TString ToString() const { - return TStringBuilder() << "{" - << " Cookie: " << Cookie - << " IsSync: " << (IsSync ? "true" : "false") - << " Partial: " << Partial - << " }"; - } - }; - - class TCacheItem { - struct TRequest { - size_t EntryIndex; - ui64 Cookie; - bool IsSync; - }; - - void Clear() { - Status.Clear(); - Kind = TNavigate::KindUnknown; - TableKind = TResolve::KindUnknown; - Created = false; - CreateStep = 0; + : Subscriber(subscriber) + , DomainOwnerId(domainOwnerId) + , Type(EType::ByPath) + , SyncCookie(0) + { + } + + TString ToString() const { + return TStringBuilder() << "{" + << " Subscriber: " << Subscriber + << " DomainOwnerId: " << DomainOwnerId + << " Type: " << static_cast<ui32>(Type) + << " SyncCookie: " << SyncCookie + << " }"; + } + + bool operator<(const TSubscriber& x) const { + return Type < x.Type; + } + + explicit operator bool() const { + return bool(Subscriber); + } + }; + + struct TResponseProps { + ui64 Cookie; + bool IsSync; + bool Partial; + + TResponseProps() + : Cookie(0) + , IsSync(false) + , Partial(false) + { + } + + explicit TResponseProps(ui64 cookie, bool isSync, bool partial) + : Cookie(cookie) + , IsSync(isSync) + , Partial(partial) + { + } + + static TResponseProps FromEvent(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { + return TResponseProps(ev->Cookie, false, false); + } + + static TResponseProps FromEvent(TSchemeBoardEvents::TEvNotifyDelete::TPtr& ev) { + return TResponseProps(ev->Cookie, false, false); + } + + static TResponseProps FromEvent(TSchemeBoardEvents::TEvSyncResponse::TPtr& ev) { + return TResponseProps(ev->Cookie, true, ev->Get()->Partial); + } + + TString ToString() const { + return TStringBuilder() << "{" + << " Cookie: " << Cookie + << " IsSync: " << (IsSync ? "true" : "false") + << " Partial: " << Partial + << " }"; + } + }; + + class TCacheItem { + struct TRequest { + size_t EntryIndex; + ui64 Cookie; + bool IsSync; + }; + + void Clear() { + Status.Clear(); + Kind = TNavigate::KindUnknown; + TableKind = TResolve::KindUnknown; + Created = false; + CreateStep = 0; // pathid is never changed (yet) so must be kept AbandonedSchemeShardsIds.clear(); - SecurityObject.Drop(); + SecurityObject.Drop(); DomainInfo.Drop(); - Attributes.clear(); - - ListNodeEntry.Drop(); - - IsPrivatePath = false; + Attributes.clear(); + + ListNodeEntry.Drop(); + + IsPrivatePath = false; // virtual must be kept - Columns.clear(); - KeyColumnTypes.clear(); + Columns.clear(); + KeyColumnTypes.clear(); NotNullColumns.clear(); - Indexes.clear(); - CdcStreams.clear(); - Partitioning.clear(); - + Indexes.clear(); + CdcStreams.clear(); + Partitioning.clear(); + Self.Drop(); DomainDescription.Drop(); - RtmrVolumeInfo.Drop(); - KesusInfo.Drop(); - SolomonVolumeInfo.Drop(); - PQGroupInfo.Drop(); + RtmrVolumeInfo.Drop(); + KesusInfo.Drop(); + SolomonVolumeInfo.Drop(); + PQGroupInfo.Drop(); OlapStoreInfo.Drop(); OlapTableInfo.Drop(); - CdcStreamInfo.Drop(); + CdcStreamInfo.Drop(); SequenceInfo.Drop(); - ReplicationInfo.Drop(); - } - + ReplicationInfo.Drop(); + } + void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) { - const auto& tableDesc = pathDesc.GetTable(); - - for (const auto& columnDesc : tableDesc.GetColumns()) { - auto& column = Columns[columnDesc.GetId()]; - column.Id = columnDesc.GetId(); - column.Name = columnDesc.GetName(); - column.PType = columnDesc.GetTypeId(); + const auto& tableDesc = pathDesc.GetTable(); + + for (const auto& columnDesc : tableDesc.GetColumns()) { + auto& column = Columns[columnDesc.GetId()]; + column.Id = columnDesc.GetId(); + column.Name = columnDesc.GetName(); + column.PType = columnDesc.GetTypeId(); if (columnDesc.GetNotNull()) { NotNullColumns.insert(columnDesc.GetName()); } - } - - KeyColumnTypes.resize(tableDesc.KeyColumnIdsSize()); - for (ui32 i : xrange(tableDesc.KeyColumnIdsSize())) { - auto* column = Columns.FindPtr(tableDesc.GetKeyColumnIds(i)); - Y_VERIFY(column != nullptr); - column->KeyOrder = i; - KeyColumnTypes[i] = column->PType; - } - - Indexes.reserve(tableDesc.TableIndexesSize()); - for (const auto& index : tableDesc.GetTableIndexes()) { - Indexes.push_back(index); - } - - CdcStreams.reserve(tableDesc.CdcStreamsSize()); - for (const auto& index : tableDesc.GetCdcStreams()) { - CdcStreams.push_back(index); - } - - if (pathDesc.TablePartitionsSize()) { - Partitioning.resize(pathDesc.TablePartitionsSize()); - for (ui32 i : xrange(pathDesc.TablePartitionsSize())) { - const auto& src = pathDesc.GetTablePartitions(i); - auto& partition = Partitioning[i]; - partition.Range = TKeyDesc::TPartitionRangeInfo(); - partition.Range->EndKeyPrefix.Parse(src.GetEndOfRangeKeyPrefix()); - partition.Range->IsInclusive = src.HasIsInclusive() && src.GetIsInclusive(); - partition.Range->IsPoint = src.HasIsPoint() && src.GetIsPoint(); - partition.ShardId = src.GetDatashardId(); - } - } - - if (pathDesc.HasDomainDescription()) { + } + + KeyColumnTypes.resize(tableDesc.KeyColumnIdsSize()); + for (ui32 i : xrange(tableDesc.KeyColumnIdsSize())) { + auto* column = Columns.FindPtr(tableDesc.GetKeyColumnIds(i)); + Y_VERIFY(column != nullptr); + column->KeyOrder = i; + KeyColumnTypes[i] = column->PType; + } + + Indexes.reserve(tableDesc.TableIndexesSize()); + for (const auto& index : tableDesc.GetTableIndexes()) { + Indexes.push_back(index); + } + + CdcStreams.reserve(tableDesc.CdcStreamsSize()); + for (const auto& index : tableDesc.GetCdcStreams()) { + CdcStreams.push_back(index); + } + + if (pathDesc.TablePartitionsSize()) { + Partitioning.resize(pathDesc.TablePartitionsSize()); + for (ui32 i : xrange(pathDesc.TablePartitionsSize())) { + const auto& src = pathDesc.GetTablePartitions(i); + auto& partition = Partitioning[i]; + partition.Range = TKeyDesc::TPartitionRangeInfo(); + partition.Range->EndKeyPrefix.Parse(src.GetEndOfRangeKeyPrefix()); + partition.Range->IsInclusive = src.HasIsInclusive() && src.GetIsInclusive(); + partition.Range->IsPoint = src.HasIsPoint() && src.GetIsPoint(); + partition.ShardId = src.GetDatashardId(); + } + } + + if (pathDesc.HasDomainDescription()) { DomainInfo = new NSchemeCache::TDomainInfo(pathDesc.GetDomainDescription()); - } - } - + } + } + void FillTableInfoFromOlapStore(const NKikimrSchemeOp::TPathDescription& pathDesc) { if (pathDesc.HasDomainDescription()) { DomainInfo = new NSchemeCache::TDomainInfo(pathDesc.GetDomainDescription()); @@ -796,551 +796,551 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } static TResolve::EKind PathSubTypeToTableKind(NKikimrSchemeOp::EPathSubType subType) { - switch (subType) { + switch (subType) { case NKikimrSchemeOp::EPathSubTypeSyncIndexImplTable: - return TResolve::KindSyncIndexTable; + return TResolve::KindSyncIndexTable; case NKikimrSchemeOp::EPathSubTypeAsyncIndexImplTable: - return TResolve::KindAsyncIndexTable; - default: - return TResolve::KindRegularTable; - } - } - + return TResolve::KindAsyncIndexTable; + default: + return TResolve::KindRegularTable; + } + } + static bool CalcPathIsPrivate(NKikimrSchemeOp::EPathType type, NKikimrSchemeOp::EPathSubType subType) { - switch (type) { + switch (type) { case NKikimrSchemeOp::EPathTypeTable: - switch (subType) { + switch (subType) { case NKikimrSchemeOp::EPathSubTypeSyncIndexImplTable: case NKikimrSchemeOp::EPathSubTypeAsyncIndexImplTable: - return true; - default: - return false; - } - case NKikimrSchemeOp::EPathTypePersQueueGroup: - switch (subType) { - case NKikimrSchemeOp::EPathSubTypeStreamImpl: - return true; - default: - return false; - } + return true; + default: + return false; + } + case NKikimrSchemeOp::EPathTypePersQueueGroup: + switch (subType) { + case NKikimrSchemeOp::EPathSubTypeStreamImpl: + return true; + default: + return false; + } case NKikimrSchemeOp::EPathTypeTableIndex: - return true; - case NKikimrSchemeOp::EPathTypeCdcStream: - return true; - default: - return false; - } - } - - static bool IsLikeDirectory(TNavigate::EKind kind) { - switch (kind) { - case TNavigate::KindSubdomain: - case TNavigate::KindPath: - case TNavigate::KindIndex: - case TNavigate::KindCdcStream: - return true; - default: - return false; - } - } - - template <typename TPtr, typename TDesc> - static void FillInfo(TNavigate::EKind kind, TPtr& ptr, TDesc&& desc) { - ptr = new typename TPtr::TValueType(); - ptr->Kind = kind; - ptr->Description = std::move(desc); - } - - // copy-paste from core/tx/scheme_cache/scheme_cache_impl.cpp - void FillRangePartitioning( - const TTableRange& range, - TVector<TKeyDesc::TPartitionInfo>& partitions - ) const { - Y_VERIFY(!Partitioning.empty()); - - partitions.clear(); - - // Temporary fix: for an empty range we need to return some datashard - // so that it can handle readset logic (send empty result to other tx participants etc.) - if (range.IsEmptyRange(KeyColumnTypes)) { - partitions.push_back(*Partitioning.begin()); - return; - } - - TVector<TKeyDesc::TPartitionInfo>::const_iterator low = LowerBound( - Partitioning.begin(), Partitioning.end(), true, - [&](const TKeyDesc::TPartitionInfo& left, bool) { - const int compares = CompareBorders<true, false>( - left.Range->EndKeyPrefix.GetCells(), range.From, - left.Range->IsInclusive || left.Range->IsPoint, - range.InclusiveFrom || range.Point, KeyColumnTypes - ); - - return (compares < 0); - } - ); - - Y_VERIFY(low != Partitioning.end(), "last key must be (inf)"); - - do { - partitions.push_back(*low); - - if (range.Point) { - return; - } - - const int prevComp = CompareBorders<true, true>( - low->Range->EndKeyPrefix.GetCells(), range.To, - low->Range->IsPoint || low->Range->IsInclusive, - range.InclusiveTo, KeyColumnTypes - ); - - if (prevComp >= 0) { - return; - } - - } while (++low != Partitioning.end()); - } - - bool IsSysTable() const { + return true; + case NKikimrSchemeOp::EPathTypeCdcStream: + return true; + default: + return false; + } + } + + static bool IsLikeDirectory(TNavigate::EKind kind) { + switch (kind) { + case TNavigate::KindSubdomain: + case TNavigate::KindPath: + case TNavigate::KindIndex: + case TNavigate::KindCdcStream: + return true; + default: + return false; + } + } + + template <typename TPtr, typename TDesc> + static void FillInfo(TNavigate::EKind kind, TPtr& ptr, TDesc&& desc) { + ptr = new typename TPtr::TValueType(); + ptr->Kind = kind; + ptr->Description = std::move(desc); + } + + // copy-paste from core/tx/scheme_cache/scheme_cache_impl.cpp + void FillRangePartitioning( + const TTableRange& range, + TVector<TKeyDesc::TPartitionInfo>& partitions + ) const { + Y_VERIFY(!Partitioning.empty()); + + partitions.clear(); + + // Temporary fix: for an empty range we need to return some datashard + // so that it can handle readset logic (send empty result to other tx participants etc.) + if (range.IsEmptyRange(KeyColumnTypes)) { + partitions.push_back(*Partitioning.begin()); + return; + } + + TVector<TKeyDesc::TPartitionInfo>::const_iterator low = LowerBound( + Partitioning.begin(), Partitioning.end(), true, + [&](const TKeyDesc::TPartitionInfo& left, bool) { + const int compares = CompareBorders<true, false>( + left.Range->EndKeyPrefix.GetCells(), range.From, + left.Range->IsInclusive || left.Range->IsPoint, + range.InclusiveFrom || range.Point, KeyColumnTypes + ); + + return (compares < 0); + } + ); + + Y_VERIFY(low != Partitioning.end(), "last key must be (inf)"); + + do { + partitions.push_back(*low); + + if (range.Point) { + return; + } + + const int prevComp = CompareBorders<true, true>( + low->Range->EndKeyPrefix.GetCells(), range.To, + low->Range->IsPoint || low->Range->IsInclusive, + range.InclusiveTo, KeyColumnTypes + ); + + if (prevComp >= 0) { + return; + } + + } while (++low != Partitioning.end()); + } + + bool IsSysTable() const { return Kind == TNavigate::KindTable && PathId.OwnerId == TSysTables::SysSchemeShard; - } - + } + void SetPathId(const TPathId pathId) { if (PathId) { Y_VERIFY(PathId == pathId); - } - + } + PathId = pathId; - } - - void SendSyncRequest() const { - Y_VERIFY(Subscriber, "it hangs if no subscriber"); - Owner->Send(Subscriber.Subscriber, new TSchemeBoardEvents::TEvSyncRequest(), 0, ++Subscriber.SyncCookie); - } - - void ResendSyncRequests(THashMap<TVariantContextPtr, TVector<TRequest>>& inFlight) const { - for (auto& [_, requests] : inFlight) { - for (auto& request : requests) { - if (!request.IsSync) { - continue; - } - - SendSyncRequest(); - request.Cookie = Subscriber.SyncCookie; - } - } - } - - template <typename TContextPtr> - void ProcessInFlightNoCheck(TContextPtr context, const TVector<TRequest>& requests) const { - // trigger LookupError - TResponseProps props(Max<ui64>(), true, true); - - for (const auto& request : requests) { - auto& entry = context->Request->ResultSet[request.EntryIndex]; - FillEntry(context.Get(), entry, props); - - Y_VERIFY(context->WaitCounter > 0); - --context->WaitCounter; - } - - if (!context->WaitCounter) { - Owner->Complete(context); - } - } - - public: - explicit TCacheItem(TSchemeCache* owner, const TSubscriber& subscriber, bool isVirtual) - : Owner(owner) - , Subscriber(subscriber) - , Filled(false) + } + + void SendSyncRequest() const { + Y_VERIFY(Subscriber, "it hangs if no subscriber"); + Owner->Send(Subscriber.Subscriber, new TSchemeBoardEvents::TEvSyncRequest(), 0, ++Subscriber.SyncCookie); + } + + void ResendSyncRequests(THashMap<TVariantContextPtr, TVector<TRequest>>& inFlight) const { + for (auto& [_, requests] : inFlight) { + for (auto& request : requests) { + if (!request.IsSync) { + continue; + } + + SendSyncRequest(); + request.Cookie = Subscriber.SyncCookie; + } + } + } + + template <typename TContextPtr> + void ProcessInFlightNoCheck(TContextPtr context, const TVector<TRequest>& requests) const { + // trigger LookupError + TResponseProps props(Max<ui64>(), true, true); + + for (const auto& request : requests) { + auto& entry = context->Request->ResultSet[request.EntryIndex]; + FillEntry(context.Get(), entry, props); + + Y_VERIFY(context->WaitCounter > 0); + --context->WaitCounter; + } + + if (!context->WaitCounter) { + Owner->Complete(context); + } + } + + public: + explicit TCacheItem(TSchemeCache* owner, const TSubscriber& subscriber, bool isVirtual) + : Owner(owner) + , Subscriber(subscriber) + , Filled(false) , Kind(TNavigate::EKind::KindUnknown) - , TableKind(TResolve::EKind::KindUnknown) + , TableKind(TResolve::EKind::KindUnknown) , Created(false) - , CreateStep(0) + , CreateStep(0) , IsPrivatePath(false) , IsVirtual(isVirtual) , SchemaVersion(0) - { - } - - TCacheItem(const TCacheItem& other) = delete; - - explicit TCacheItem(TCacheItem&& other) - : Owner(other.Owner) - , Subscriber(other.Subscriber) - , Filled(other.Filled) - , InFlight(std::move(other.InFlight)) - , Kind(other.Kind) - , TableKind(other.TableKind) - , Created(other.Created) - , CreateStep(other.CreateStep) - , PathId(other.PathId) + { + } + + TCacheItem(const TCacheItem& other) = delete; + + explicit TCacheItem(TCacheItem&& other) + : Owner(other.Owner) + , Subscriber(other.Subscriber) + , Filled(other.Filled) + , InFlight(std::move(other.InFlight)) + , Kind(other.Kind) + , TableKind(other.TableKind) + , Created(other.Created) + , CreateStep(other.CreateStep) + , PathId(other.PathId) , IsPrivatePath(other.IsPrivatePath) , IsVirtual(other.IsVirtual) , SchemaVersion(other.SchemaVersion) - { - if (other.Subscriber) { - other.Subscriber = TSubscriber(); - } - } - - TString ToString() const { - return TStringBuilder() << "{" - << " Subscriber: " << Subscriber.ToString() - << " Filled: " << Filled + { + if (other.Subscriber) { + other.Subscriber = TSubscriber(); + } + } + + TString ToString() const { + return TStringBuilder() << "{" + << " Subscriber: " << Subscriber.ToString() + << " Filled: " << Filled << " Status: " << (Status ? NKikimrScheme::EStatus_Name(*Status) : "undefined") - << " Kind: " << static_cast<ui32>(Kind) - << " TableKind: " << static_cast<ui32>(TableKind) - << " Created: " << Created - << " CreateStep: " << CreateStep - << " PathId: " << PathId - << " DomainId: " << GetDomainId() - << " IsPrivatePath: " << IsPrivatePath - << " IsVirtual: " << IsVirtual - << " SchemaVersion: " << SchemaVersion - << " }"; - } - - template <typename T> - static TString ProtoJsonString(const T& message) { - using namespace google::protobuf::util; - - JsonPrintOptions opts; - opts.preserve_proto_field_names = true; - - TString jsonString; - MessageToJsonString(message, &jsonString, opts); - - return jsonString; - } - - TString ToJsonString() const { - NJsonWriter::TBuf json; - auto root = json.BeginObject(); - - root.WriteKey("Subscriber").BeginObject() - .WriteKey("ActorId").WriteString(::ToString(Subscriber.Subscriber)) - .WriteKey("DomainOwnerId").WriteULongLong(Subscriber.DomainOwnerId) - .WriteKey("Type").WriteInt(static_cast<int>(Subscriber.Type)) - .WriteKey("SyncCookie").WriteULongLong(Subscriber.SyncCookie) - .EndObject(); - - root.WriteKey("Filled").WriteBool(Filled); - root.WriteKey("Path").WriteString(Path); - root.WriteKey("PathId").WriteString(::ToString(PathId)); - root.WriteKey("Kind").WriteInt(static_cast<int>(Kind)); - root.WriteKey("TableKind").WriteInt(static_cast<int>(TableKind)); - root.WriteKey("Created").WriteBool(Created); - root.WriteKey("CreateStep").WriteULongLong(CreateStep); - root.WriteKey("IsPrivatePath").WriteBool(IsPrivatePath); - root.WriteKey("IsVirtual").WriteBool(IsVirtual); - root.WriteKey("SchemaVersion").WriteULongLong(SchemaVersion); - - if (Status) { + << " Kind: " << static_cast<ui32>(Kind) + << " TableKind: " << static_cast<ui32>(TableKind) + << " Created: " << Created + << " CreateStep: " << CreateStep + << " PathId: " << PathId + << " DomainId: " << GetDomainId() + << " IsPrivatePath: " << IsPrivatePath + << " IsVirtual: " << IsVirtual + << " SchemaVersion: " << SchemaVersion + << " }"; + } + + template <typename T> + static TString ProtoJsonString(const T& message) { + using namespace google::protobuf::util; + + JsonPrintOptions opts; + opts.preserve_proto_field_names = true; + + TString jsonString; + MessageToJsonString(message, &jsonString, opts); + + return jsonString; + } + + TString ToJsonString() const { + NJsonWriter::TBuf json; + auto root = json.BeginObject(); + + root.WriteKey("Subscriber").BeginObject() + .WriteKey("ActorId").WriteString(::ToString(Subscriber.Subscriber)) + .WriteKey("DomainOwnerId").WriteULongLong(Subscriber.DomainOwnerId) + .WriteKey("Type").WriteInt(static_cast<int>(Subscriber.Type)) + .WriteKey("SyncCookie").WriteULongLong(Subscriber.SyncCookie) + .EndObject(); + + root.WriteKey("Filled").WriteBool(Filled); + root.WriteKey("Path").WriteString(Path); + root.WriteKey("PathId").WriteString(::ToString(PathId)); + root.WriteKey("Kind").WriteInt(static_cast<int>(Kind)); + root.WriteKey("TableKind").WriteInt(static_cast<int>(TableKind)); + root.WriteKey("Created").WriteBool(Created); + root.WriteKey("CreateStep").WriteULongLong(CreateStep); + root.WriteKey("IsPrivatePath").WriteBool(IsPrivatePath); + root.WriteKey("IsVirtual").WriteBool(IsVirtual); + root.WriteKey("SchemaVersion").WriteULongLong(SchemaVersion); + + if (Status) { root.WriteKey("Status").WriteString(NKikimrScheme::EStatus_Name(*Status)); - } else { - root.WriteKey("Status").WriteNull(); - } - - if (Attributes) { - auto attrs = root.WriteKey("Attributes").BeginObject(); - - for (const auto& [key, value] : Attributes) { - attrs.WriteKey(key).WriteString(value); - } - - attrs.EndObject(); - } - - if (AbandonedSchemeShardsIds) { - auto abandoned = root.WriteKey("AbandonedSchemeShardsIds").BeginList(); - - for (const auto ssId : AbandonedSchemeShardsIds) { - abandoned.WriteULongLong(ssId); - } - - abandoned.EndList(); - } - - if (DomainInfo) { - root.WriteKey("DomainInfo").BeginObject() - .WriteKey("DomainKey").WriteString(::ToString(DomainInfo->DomainKey)) - .WriteKey("ResourcesDomainKey").WriteString(::ToString(DomainInfo->ResourcesDomainKey)) - .WriteKey("Params").UnsafeWriteValue(ProtoJsonString(DomainInfo->Params)) - .EndObject(); - } - - if (Self) { - root.WriteKey("Self").UnsafeWriteValue(ProtoJsonString(Self->Info)); - } - - if (ListNodeEntry) { - auto children = root.WriteKey("Children").BeginList(); - - for (const auto& child : ListNodeEntry->Children) { - children.BeginObject() - .WriteKey("Name").WriteString(child.Name) - .WriteKey("PathId").WriteString(::ToString(child.PathId)) - .WriteKey("SchemaVersion").WriteULongLong(child.SchemaVersion) - .WriteKey("Kind").WriteInt(static_cast<int>(child.Kind)) - .EndObject(); - } - - children.EndList(); - } - - if (Columns) { - auto columns = root.WriteKey("Columns").BeginList(); - - for (const auto& [_, column] : Columns) { - columns.BeginObject() - .WriteKey("Id").WriteULongLong(column.Id) - .WriteKey("Name").WriteString(column.Name) - .WriteKey("Type").WriteULongLong(column.PType) - .WriteKey("KeyOrder").WriteInt(column.KeyOrder) - .EndObject(); - } - - columns.EndList(); - } - - #define DESCRIPTION_PART(name) \ - if (name) { \ - root.WriteKey(#name).UnsafeWriteValue(ProtoJsonString(name->Description)); \ - } - - DESCRIPTION_PART(DomainDescription) - DESCRIPTION_PART(RtmrVolumeInfo) - DESCRIPTION_PART(KesusInfo); - DESCRIPTION_PART(SolomonVolumeInfo); - DESCRIPTION_PART(PQGroupInfo); - DESCRIPTION_PART(OlapStoreInfo); - DESCRIPTION_PART(OlapTableInfo); - DESCRIPTION_PART(CdcStreamInfo); - DESCRIPTION_PART(SequenceInfo); - DESCRIPTION_PART(ReplicationInfo); - - #undef DESCRIPTION_PART - - root.EndObject(); - return json.Str(); - } - - void OnEvict(bool respondInFlight = true) { - if (Subscriber) { - Owner->Send(Subscriber.Subscriber, new TEvents::TEvPoisonPill()); - Subscriber = TSubscriber(); - } - - if (!respondInFlight) { - return; - } - - for (const auto& [contextVariant, requests] : InFlight) { + } else { + root.WriteKey("Status").WriteNull(); + } + + if (Attributes) { + auto attrs = root.WriteKey("Attributes").BeginObject(); + + for (const auto& [key, value] : Attributes) { + attrs.WriteKey(key).WriteString(value); + } + + attrs.EndObject(); + } + + if (AbandonedSchemeShardsIds) { + auto abandoned = root.WriteKey("AbandonedSchemeShardsIds").BeginList(); + + for (const auto ssId : AbandonedSchemeShardsIds) { + abandoned.WriteULongLong(ssId); + } + + abandoned.EndList(); + } + + if (DomainInfo) { + root.WriteKey("DomainInfo").BeginObject() + .WriteKey("DomainKey").WriteString(::ToString(DomainInfo->DomainKey)) + .WriteKey("ResourcesDomainKey").WriteString(::ToString(DomainInfo->ResourcesDomainKey)) + .WriteKey("Params").UnsafeWriteValue(ProtoJsonString(DomainInfo->Params)) + .EndObject(); + } + + if (Self) { + root.WriteKey("Self").UnsafeWriteValue(ProtoJsonString(Self->Info)); + } + + if (ListNodeEntry) { + auto children = root.WriteKey("Children").BeginList(); + + for (const auto& child : ListNodeEntry->Children) { + children.BeginObject() + .WriteKey("Name").WriteString(child.Name) + .WriteKey("PathId").WriteString(::ToString(child.PathId)) + .WriteKey("SchemaVersion").WriteULongLong(child.SchemaVersion) + .WriteKey("Kind").WriteInt(static_cast<int>(child.Kind)) + .EndObject(); + } + + children.EndList(); + } + + if (Columns) { + auto columns = root.WriteKey("Columns").BeginList(); + + for (const auto& [_, column] : Columns) { + columns.BeginObject() + .WriteKey("Id").WriteULongLong(column.Id) + .WriteKey("Name").WriteString(column.Name) + .WriteKey("Type").WriteULongLong(column.PType) + .WriteKey("KeyOrder").WriteInt(column.KeyOrder) + .EndObject(); + } + + columns.EndList(); + } + + #define DESCRIPTION_PART(name) \ + if (name) { \ + root.WriteKey(#name).UnsafeWriteValue(ProtoJsonString(name->Description)); \ + } + + DESCRIPTION_PART(DomainDescription) + DESCRIPTION_PART(RtmrVolumeInfo) + DESCRIPTION_PART(KesusInfo); + DESCRIPTION_PART(SolomonVolumeInfo); + DESCRIPTION_PART(PQGroupInfo); + DESCRIPTION_PART(OlapStoreInfo); + DESCRIPTION_PART(OlapTableInfo); + DESCRIPTION_PART(CdcStreamInfo); + DESCRIPTION_PART(SequenceInfo); + DESCRIPTION_PART(ReplicationInfo); + + #undef DESCRIPTION_PART + + root.EndObject(); + return json.Str(); + } + + void OnEvict(bool respondInFlight = true) { + if (Subscriber) { + Owner->Send(Subscriber.Subscriber, new TEvents::TEvPoisonPill()); + Subscriber = TSubscriber(); + } + + if (!respondInFlight) { + return; + } + + for (const auto& [contextVariant, requests] : InFlight) { if (auto* context = std::get_if<TNavigateContextPtr>(&contextVariant)) { - ProcessInFlightNoCheck(*context, requests); + ProcessInFlightNoCheck(*context, requests); } else if (auto* context = std::get_if<TResolveContextPtr>(&contextVariant)) { - ProcessInFlightNoCheck(*context, requests); - } else { - Y_FAIL("unknown context type"); - } - } - } - - TCacheItem& Merge(TCacheItem&& other) noexcept { - if (Subscriber < other.Subscriber) { - SwapSubscriber(other.Subscriber); - } else { - ResendSyncRequests(other.InFlight); - } - - InFlight.insert(other.InFlight.begin(), other.InFlight.end()); - other.OnEvict(false); - - return *this; - } - - void SwapSubscriber(TSubscriber& subscriber) { - std::swap(Subscriber, subscriber); - ResendSyncRequests(InFlight); - } - - void MoveInFlightNavigateByPathRequests(TCacheItem& recipient) { - EraseNodesIf(InFlight, [&recipient](auto& kv) { + ProcessInFlightNoCheck(*context, requests); + } else { + Y_FAIL("unknown context type"); + } + } + } + + TCacheItem& Merge(TCacheItem&& other) noexcept { + if (Subscriber < other.Subscriber) { + SwapSubscriber(other.Subscriber); + } else { + ResendSyncRequests(other.InFlight); + } + + InFlight.insert(other.InFlight.begin(), other.InFlight.end()); + other.OnEvict(false); + + return *this; + } + + void SwapSubscriber(TSubscriber& subscriber) { + std::swap(Subscriber, subscriber); + ResendSyncRequests(InFlight); + } + + void MoveInFlightNavigateByPathRequests(TCacheItem& recipient) { + EraseNodesIf(InFlight, [&recipient](auto& kv) { auto* context = std::get_if<TNavigateContextPtr>(&kv.first); - if (!context) { - return false; - } - - EraseIf(kv.second, [context, &recipient](const TRequest& request) { - const auto& entry = context->Get()->Request->ResultSet[request.EntryIndex]; - if (entry.RequestType != TNavigate::TEntry::ERequestType::ByPath) { - return false; - } - - Y_VERIFY(context->Get()->WaitCounter > 0); - --context->Get()->WaitCounter; - - recipient.AddInFlight(*context, request.EntryIndex, request.IsSync); - return true; - }); - - return kv.second.empty(); - }); - } - - const TSubscriber& GetSubcriber() const { - return Subscriber; - } - - template <typename TContextPtr> + if (!context) { + return false; + } + + EraseIf(kv.second, [context, &recipient](const TRequest& request) { + const auto& entry = context->Get()->Request->ResultSet[request.EntryIndex]; + if (entry.RequestType != TNavigate::TEntry::ERequestType::ByPath) { + return false; + } + + Y_VERIFY(context->Get()->WaitCounter > 0); + --context->Get()->WaitCounter; + + recipient.AddInFlight(*context, request.EntryIndex, request.IsSync); + return true; + }); + + return kv.second.empty(); + }); + } + + const TSubscriber& GetSubcriber() const { + return Subscriber; + } + + template <typename TContextPtr> void AddInFlight(TContextPtr context, const size_t entryIndex, const bool isSync) const { - if (IsVirtual) { + if (IsVirtual) { return; - } - - if (isSync) { - SendSyncRequest(); - } - - auto it = InFlight.find(context); - if (it == InFlight.end()) { - it = InFlight.emplace(context, TVector<TRequest>()).first; - } - - it->second.push_back({entryIndex, Subscriber.SyncCookie, isSync}); - ++context->WaitCounter; - } - - template <typename TContext> - void ProcessInFlight(TContext* context, TVector<TRequest>& requests, const TResponseProps& response) const { - EraseIf(requests, [this, context, response](const TRequest& request) { - if (request.IsSync != response.IsSync) { - return false; - } - - if (request.IsSync && request.Cookie > response.Cookie) { - return false; - } - - auto& entry = context->Request->ResultSet[request.EntryIndex]; - if (!entry.SyncVersion || (entry.SyncVersion && response.IsSync)) { - FillEntry(context, entry, response); - } - - Y_VERIFY(context->WaitCounter > 0); - --context->WaitCounter; - - return true; - }); - } - - TVector<TVariantContextPtr> ProcessInFlight(const TResponseProps& response) const { - TVector<TVariantContextPtr> processed(Reserve(InFlight.size())); - - EraseNodesIf(InFlight, [this, &processed, response](auto& kv) { - processed.push_back(kv.first); - + } + + if (isSync) { + SendSyncRequest(); + } + + auto it = InFlight.find(context); + if (it == InFlight.end()) { + it = InFlight.emplace(context, TVector<TRequest>()).first; + } + + it->second.push_back({entryIndex, Subscriber.SyncCookie, isSync}); + ++context->WaitCounter; + } + + template <typename TContext> + void ProcessInFlight(TContext* context, TVector<TRequest>& requests, const TResponseProps& response) const { + EraseIf(requests, [this, context, response](const TRequest& request) { + if (request.IsSync != response.IsSync) { + return false; + } + + if (request.IsSync && request.Cookie > response.Cookie) { + return false; + } + + auto& entry = context->Request->ResultSet[request.EntryIndex]; + if (!entry.SyncVersion || (entry.SyncVersion && response.IsSync)) { + FillEntry(context, entry, response); + } + + Y_VERIFY(context->WaitCounter > 0); + --context->WaitCounter; + + return true; + }); + } + + TVector<TVariantContextPtr> ProcessInFlight(const TResponseProps& response) const { + TVector<TVariantContextPtr> processed(Reserve(InFlight.size())); + + EraseNodesIf(InFlight, [this, &processed, response](auto& kv) { + processed.push_back(kv.first); + if (auto* context = std::get_if<TNavigateContextPtr>(&kv.first)) { - ProcessInFlight(context->Get(), kv.second, response); + ProcessInFlight(context->Get(), kv.second, response); } else if (auto* context = std::get_if<TResolveContextPtr>(&kv.first)) { - ProcessInFlight(context->Get(), kv.second, response); - } else { - Y_FAIL("unknown context type"); - } - - return kv.second.empty(); - }); - - return processed; - } - - void FillAsSysPath() { - Clear(); - Filled = true; - + ProcessInFlight(context->Get(), kv.second, response); + } else { + Y_FAIL("unknown context type"); + } + + return kv.second.empty(); + }); + + return processed; + } + + void FillAsSysPath() { + Clear(); + Filled = true; + Status = NKikimrScheme::StatusSuccess; - Kind = TNavigate::KindPath; - Created = true; + Kind = TNavigate::KindPath; + Created = true; PathId = TPathId(TSysTables::SysSchemeShard, 0); Path = "/sys"; IsVirtual = true; - } - - void FillAsSysLocks(const bool v2) { - Clear(); - Filled = true; - + } + + void FillAsSysLocks(const bool v2) { + Clear(); + Filled = true; + Status = NKikimrScheme::StatusSuccess; - Kind = TNavigate::KindTable; - Created = true; - PathId = TPathId(TSysTables::SysSchemeShard, v2 ? TSysTables::SysTableLocks2 : TSysTables::SysTableLocks); + Kind = TNavigate::KindTable; + Created = true; + PathId = TPathId(TSysTables::SysSchemeShard, v2 ? TSysTables::SysTableLocks2 : TSysTables::SysTableLocks); Path = v2 ? "/sys/locks2" : "/sys/locks"; - - TVector<ui32> keyColumnTypes; - TSysTables::TLocksTable::GetInfo(Columns, keyColumnTypes, v2); - for (ui32 type : keyColumnTypes) { - KeyColumnTypes.push_back(type); - } + + TVector<ui32> keyColumnTypes; + TSysTables::TLocksTable::GetInfo(Columns, keyColumnTypes, v2); + for (ui32 type : keyColumnTypes) { + KeyColumnTypes.push_back(type); + } IsPrivatePath = true; IsVirtual = true; - } - - void Fill(TSchemeBoardEvents::TEvNotifyUpdate& notify) { - Clear(); - Filled = true; - - Y_VERIFY(notify.PathId); - SetPathId(notify.PathId); - - if (!notify.DescribeSchemeResult.HasStatus()) { - return; - } else { - Status = notify.DescribeSchemeResult.GetStatus(); + } + + void Fill(TSchemeBoardEvents::TEvNotifyUpdate& notify) { + Clear(); + Filled = true; + + Y_VERIFY(notify.PathId); + SetPathId(notify.PathId); + + if (!notify.DescribeSchemeResult.HasStatus()) { + return; + } else { + Status = notify.DescribeSchemeResult.GetStatus(); if (Status != NKikimrScheme::StatusSuccess) { - return; - } - } - - Y_VERIFY(notify.DescribeSchemeResult.HasPathDescription()); - auto& pathDesc = *notify.DescribeSchemeResult.MutablePathDescription(); - + return; + } + } + + Y_VERIFY(notify.DescribeSchemeResult.HasPathDescription()); + auto& pathDesc = *notify.DescribeSchemeResult.MutablePathDescription(); + Y_VERIFY(notify.DescribeSchemeResult.HasPath()); Path = notify.DescribeSchemeResult.GetPath(); - const auto& abandoned = pathDesc.GetAbandonedTenantsSchemeShards(); - AbandonedSchemeShardsIds = TSet<ui64>(abandoned.begin(), abandoned.end()); + const auto& abandoned = pathDesc.GetAbandonedTenantsSchemeShards(); + AbandonedSchemeShardsIds = TSet<ui64>(abandoned.begin(), abandoned.end()); - Y_VERIFY(pathDesc.HasSelf()); - const auto& entryDesc = pathDesc.GetSelf(); + Y_VERIFY(pathDesc.HasSelf()); + const auto& entryDesc = pathDesc.GetSelf(); Self = new TNavigate::TDirEntryInfo(); Self->Info.CopyFrom(entryDesc); - - Created = entryDesc.HasCreateFinished() && entryDesc.GetCreateFinished(); - CreateStep = entryDesc.GetCreateStep(); - SecurityObject = new TSecurityObject(entryDesc.GetOwner(), entryDesc.GetEffectiveACL(), false); + + Created = entryDesc.HasCreateFinished() && entryDesc.GetCreateFinished(); + CreateStep = entryDesc.GetCreateStep(); + SecurityObject = new TSecurityObject(entryDesc.GetOwner(), entryDesc.GetEffectiveACL(), false); DomainInfo = new NSchemeCache::TDomainInfo(pathDesc.GetDomainDescription()); - - for (const auto& attr : pathDesc.GetUserAttributes()) { - Attributes[attr.GetKey()] = attr.GetValue(); - } - - auto tableSchemaVersion = [](const auto& entryDesc) { - return entryDesc.HasVersion() ? entryDesc.GetVersion().GetTableSchemaVersion() : 0; - }; - auto tableIndexVersion = [](const auto& entryDesc) { - return entryDesc.HasVersion() ? entryDesc.GetVersion().GetTableIndexVersion() : 0; - }; - - switch (entryDesc.GetPathType()) { + + for (const auto& attr : pathDesc.GetUserAttributes()) { + Attributes[attr.GetKey()] = attr.GetValue(); + } + + auto tableSchemaVersion = [](const auto& entryDesc) { + return entryDesc.HasVersion() ? entryDesc.GetVersion().GetTableSchemaVersion() : 0; + }; + auto tableIndexVersion = [](const auto& entryDesc) { + return entryDesc.HasVersion() ? entryDesc.GetVersion().GetTableIndexVersion() : 0; + }; + + switch (entryDesc.GetPathType()) { case NKikimrSchemeOp::EPathTypeSubDomain: - Kind = TNavigate::KindSubdomain; + Kind = TNavigate::KindSubdomain; FillInfo(Kind, DomainDescription, std::move(*pathDesc.MutableDomainDescription())); - break; + break; case NKikimrSchemeOp::EPathTypeExtSubDomain: Kind = TNavigate::KindExtSubdomain; FillInfo(Kind, DomainDescription, std::move(*pathDesc.MutableDomainDescription())); @@ -1348,20 +1348,20 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { case NKikimrSchemeOp::EPathTypeDir: case NKikimrSchemeOp::EPathTypeBlockStoreVolume: case NKikimrSchemeOp::EPathTypeFileStore: - Kind = TNavigate::KindPath; + Kind = TNavigate::KindPath; if (entryDesc.GetPathId() == entryDesc.GetParentPathId()) { FillInfo(Kind, DomainDescription, std::move(*pathDesc.MutableDomainDescription())); } - break; + break; case NKikimrSchemeOp::EPathTypeTable: - Kind = TNavigate::KindTable; - TableKind = PathSubTypeToTableKind(entryDesc.GetPathSubType()); - IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); - if (Created) { - FillTableInfo(pathDesc); - SchemaVersion = tableSchemaVersion(entryDesc); - } - break; + Kind = TNavigate::KindTable; + TableKind = PathSubTypeToTableKind(entryDesc.GetPathSubType()); + IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); + if (Created) { + FillTableInfo(pathDesc); + SchemaVersion = tableSchemaVersion(entryDesc); + } + break; case NKikimrSchemeOp::EPathTypeColumnStore: Kind = TNavigate::KindOlapStore; if (Created) { @@ -1381,71 +1381,71 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } break; case NKikimrSchemeOp::EPathTypeTableIndex: - Kind = TNavigate::KindIndex; - IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); - SchemaVersion = tableIndexVersion(entryDesc); - break; + Kind = TNavigate::KindIndex; + IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); + SchemaVersion = tableIndexVersion(entryDesc); + break; case NKikimrSchemeOp::EPathTypeRtmrVolume: - Kind = TNavigate::KindRtmr; - FillInfo(Kind, RtmrVolumeInfo, std::move(*pathDesc.MutableRtmrVolumeDescription())); - break; + Kind = TNavigate::KindRtmr; + FillInfo(Kind, RtmrVolumeInfo, std::move(*pathDesc.MutableRtmrVolumeDescription())); + break; case NKikimrSchemeOp::EPathTypeKesus: - Kind = TNavigate::KindKesus; - FillInfo(Kind, KesusInfo, std::move(*pathDesc.MutableKesus())); - break; + Kind = TNavigate::KindKesus; + FillInfo(Kind, KesusInfo, std::move(*pathDesc.MutableKesus())); + break; case NKikimrSchemeOp::EPathTypeSolomonVolume: - Kind = TNavigate::KindSolomon; - FillInfo(Kind, SolomonVolumeInfo, std::move(*pathDesc.MutableSolomonDescription())); - break; + Kind = TNavigate::KindSolomon; + FillInfo(Kind, SolomonVolumeInfo, std::move(*pathDesc.MutableSolomonDescription())); + break; case NKikimrSchemeOp::EPathTypePersQueueGroup: - Kind = TNavigate::KindTopic; - IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); - FillInfo(Kind, PQGroupInfo, std::move(*pathDesc.MutablePersQueueGroup())); - break; + Kind = TNavigate::KindTopic; + IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); + FillInfo(Kind, PQGroupInfo, std::move(*pathDesc.MutablePersQueueGroup())); + break; case NKikimrSchemeOp::EPathTypeCdcStream: - Kind = TNavigate::KindCdcStream; - IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); - FillInfo(Kind, CdcStreamInfo, std::move(*pathDesc.MutableCdcStreamDescription())); - if (CdcStreamInfo->Description.HasPathId()) { - const auto& pathId = CdcStreamInfo->Description.GetPathId(); - CdcStreamInfo->PathId = TPathId(pathId.GetOwnerId(), pathId.GetLocalId()); - } - break; + Kind = TNavigate::KindCdcStream; + IsPrivatePath = CalcPathIsPrivate(entryDesc.GetPathType(), entryDesc.GetPathSubType()); + FillInfo(Kind, CdcStreamInfo, std::move(*pathDesc.MutableCdcStreamDescription())); + if (CdcStreamInfo->Description.HasPathId()) { + const auto& pathId = CdcStreamInfo->Description.GetPathId(); + CdcStreamInfo->PathId = TPathId(pathId.GetOwnerId(), pathId.GetLocalId()); + } + break; case NKikimrSchemeOp::EPathTypeSequence: Kind = TNavigate::KindSequence; FillInfo(Kind, SequenceInfo, std::move(*pathDesc.MutableSequenceDescription())); break; - case NKikimrSchemeOp::EPathTypeReplication: - Kind = TNavigate::KindReplication; - FillInfo(Kind, ReplicationInfo, std::move(*pathDesc.MutableReplicationDescription())); - break; + case NKikimrSchemeOp::EPathTypeReplication: + Kind = TNavigate::KindReplication; + FillInfo(Kind, ReplicationInfo, std::move(*pathDesc.MutableReplicationDescription())); + break; case NKikimrSchemeOp::EPathTypeInvalid: - Y_VERIFY_DEBUG(false, "Invalid path type"); + Y_VERIFY_DEBUG(false, "Invalid path type"); break; - } - - if (IsLikeDirectory(Kind)) { - ListNodeEntry = new TNavigate::TListNodeEntry(); - ListNodeEntry->Kind = TNavigate::KindPath; - ListNodeEntry->Children.reserve(pathDesc.ChildrenSize()); - - for (const auto& child : pathDesc.GetChildren()) { + } + + if (IsLikeDirectory(Kind)) { + ListNodeEntry = new TNavigate::TListNodeEntry(); + ListNodeEntry->Kind = TNavigate::KindPath; + ListNodeEntry->Children.reserve(pathDesc.ChildrenSize()); + + for (const auto& child : pathDesc.GetChildren()) { const auto& name = child.GetName(); const auto pathId = TPathId(child.GetSchemeshardId(), child.GetPathId()); - switch (child.GetPathType()) { + switch (child.GetPathType()) { case NKikimrSchemeOp::EPathTypeSubDomain: case NKikimrSchemeOp::EPathTypeDir: case NKikimrSchemeOp::EPathTypeBlockStoreVolume: case NKikimrSchemeOp::EPathTypeFileStore: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindPath); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindPath); + break; case NKikimrSchemeOp::EPathTypeExtSubDomain: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindExtSubdomain); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindExtSubdomain); + break; case NKikimrSchemeOp::EPathTypeTable: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindTable, tableSchemaVersion(child)); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindTable, tableSchemaVersion(child)); + break; case NKikimrSchemeOp::EPathTypeColumnStore: ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindOlapStore); break; @@ -1453,60 +1453,60 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindOlapTable); break; case NKikimrSchemeOp::EPathTypeRtmrVolume: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindRtmr); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindRtmr); + break; case NKikimrSchemeOp::EPathTypeKesus: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindKesus); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindKesus); + break; case NKikimrSchemeOp::EPathTypeSolomonVolume: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindSolomon); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindSolomon); + break; case NKikimrSchemeOp::EPathTypePersQueueGroup: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindTopic); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindTopic); + break; case NKikimrSchemeOp::EPathTypeCdcStream: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindCdcStream); - break; + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindCdcStream); + break; case NKikimrSchemeOp::EPathTypeSequence: ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindSequence); break; - case NKikimrSchemeOp::EPathTypeReplication: - ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindReplication); - break; + case NKikimrSchemeOp::EPathTypeReplication: + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindReplication); + break; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeInvalid: - Y_VERIFY_DEBUG(false, "Invalid path type"); + Y_VERIFY_DEBUG(false, "Invalid path type"); break; - } - } - } - } - - void Fill(TSchemeBoardEvents::TEvNotifyDelete& notify) { - Clear(); - Filled = true; - - if (notify.PathId) { - SetPathId(notify.PathId); - } - - Y_VERIFY_DEBUG(Subscriber.DomainOwnerId); - if (notify.Strong) { + } + } + } + } + + void Fill(TSchemeBoardEvents::TEvNotifyDelete& notify) { + Clear(); + Filled = true; + + if (notify.PathId) { + SetPathId(notify.PathId); + } + + Y_VERIFY_DEBUG(Subscriber.DomainOwnerId); + if (notify.Strong) { Status = NKikimrScheme::StatusPathDoesNotExist; - } - } - - void Fill(TSchemeBoardEvents::TEvSyncResponse&) { - } - - bool IsFilled() const { - return Filled; - } - + } + } + + void Fill(TSchemeBoardEvents::TEvSyncResponse&) { + } + + bool IsFilled() const { + return Filled; + } + TMaybe<NKikimrScheme::EStatus> GetStatus() const { - return Status; - } - + return Status; + } + TPathId GetPathId() const { return IsFilled() ? PathId : TPathId(); } @@ -1515,10 +1515,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return (IsFilled() && DomainInfo) ? DomainInfo->DomainKey : TDomainId(); } - auto GetDomainInfo() const { - return DomainInfo; - } - + auto GetDomainInfo() const { + return DomainInfo; + } + const TSet<ui64>& GetAbandonedSchemeShardIds() const { return AbandonedSchemeShardsIds; } @@ -1530,7 +1530,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { if (sysViewInfo == NSysView::SysPathName) { if (entry.Operation == TNavigate::OpTable) { - return SetError(context, entry, TNavigate::EStatus::PathNotTable); + return SetError(context, entry, TNavigate::EStatus::PathNotTable); } auto listNodeEntry = MakeIntrusive<TNavigate::TListNodeEntry>(); @@ -1551,7 +1551,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } else { auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); if (!schema) { - return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); } entry.Kind = TNavigate::KindTable; @@ -1579,33 +1579,33 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.DomainInfo = DomainInfo; } - void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = TResponseProps()) const { + void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = TResponseProps()) const { SBC_LOG_D("FillEntry for TNavigate" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); - - if (props.IsSync && props.Partial) { - return SetError(context, entry, TNavigate::EStatus::LookupError); - } - + << ": self# " << Owner->SelfId() + << ", cacheItem# " << ToString() + << ", entry# " << entry.ToString() + << ", props# " << props.ToString()); + + if (props.IsSync && props.Partial) { + return SetError(context, entry, TNavigate::EStatus::LookupError); + } + if (Status && Status == NKikimrScheme::StatusPathDoesNotExist) { - return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - + return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + } + if (!Status || Status != NKikimrScheme::StatusSuccess) { - return SetError(context, entry, TNavigate::EStatus::LookupError); - } - + return SetError(context, entry, TNavigate::EStatus::LookupError); + } + if (!entry.TableId.SysViewInfo.empty()) { if (Kind == TNavigate::KindPath) { auto split = SplitPath(Path); if (split.size() == 1) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); } } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); entry.OlapStoreInfo = OlapStoreInfo; @@ -1616,37 +1616,37 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.OlapTableInfo = OlapTableInfo; return; } - - return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + + return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); } const bool isTable = Kind == TNavigate::KindTable || Kind == TNavigate::KindOlapTable; if (entry.Operation == TNavigate::OpTable && !isTable) { - return SetError(context, entry, TNavigate::EStatus::PathNotTable); - } - + return SetError(context, entry, TNavigate::EStatus::PathNotTable); + } + if (!Created && isTable) { - return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - - if (!entry.ShowPrivatePath && IsPrivatePath) { - return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); - } - - // common + return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + } + + if (!entry.ShowPrivatePath && IsPrivatePath) { + return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown); + } + + // common entry.SecurityObject = SecurityObject; entry.DomainInfo = DomainInfo; - entry.Attributes = Attributes; + entry.Attributes = Attributes; entry.Status = TNavigate::EStatus::Ok; if (Kind == TNavigate::KindExtSubdomain && entry.RedirectRequired) { - SetError(context, entry, TNavigate::EStatus::RedirectLookupError); + SetError(context, entry, TNavigate::EStatus::RedirectLookupError); } - entry.Kind = Kind; - entry.CreateStep = CreateStep; - + entry.Kind = Kind; + entry.CreateStep = CreateStep; + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { if (Kind == TNavigate::KindTable) { entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, SchemaVersion); @@ -1660,90 +1660,90 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } - if (entry.Operation == TNavigate::OpList) { - entry.ListNodeEntry = ListNodeEntry; - } - - // specific (it's safe to fill them all) + if (entry.Operation == TNavigate::OpList) { + entry.ListNodeEntry = ListNodeEntry; + } + + // specific (it's safe to fill them all) entry.Self = Self; - entry.Columns = Columns; + entry.Columns = Columns; entry.NotNullColumns = NotNullColumns; - entry.Indexes = Indexes; - entry.CdcStreams = CdcStreams; + entry.Indexes = Indexes; + entry.CdcStreams = CdcStreams; entry.DomainDescription = DomainDescription; - entry.RTMRVolumeInfo = RtmrVolumeInfo; - entry.KesusInfo = KesusInfo; - entry.SolomonVolumeInfo = SolomonVolumeInfo; - entry.PQGroupInfo = PQGroupInfo; + entry.RTMRVolumeInfo = RtmrVolumeInfo; + entry.KesusInfo = KesusInfo; + entry.SolomonVolumeInfo = SolomonVolumeInfo; + entry.PQGroupInfo = PQGroupInfo; entry.OlapStoreInfo = OlapStoreInfo; entry.OlapTableInfo = OlapTableInfo; - entry.CdcStreamInfo = CdcStreamInfo; + entry.CdcStreamInfo = CdcStreamInfo; entry.SequenceInfo = SequenceInfo; - entry.ReplicationInfo = ReplicationInfo; - } - - bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, + entry.ReplicationInfo = ReplicationInfo; + } + + bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, const TVector<NScheme::TTypeId>& keyColumnTypes, const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const { - TKeyDesc& keyDesc = *entry.KeyDescription; - + TKeyDesc& keyDesc = *entry.KeyDescription; + // check key types if (keyDesc.KeyColumnTypes.size() > keyColumnTypes.size()) { - SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); + SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); return false; - } - - for (ui32 i : xrange(keyDesc.KeyColumnTypes.size())) { + } + + for (ui32 i : xrange(keyDesc.KeyColumnTypes.size())) { if (keyDesc.KeyColumnTypes[i] != keyColumnTypes[i]) { - SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); + SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed); return false; - } - } - + } + } + // check operations - keyDesc.ColumnInfos.clear(); - keyDesc.ColumnInfos.reserve(keyDesc.Columns.size()); - for (auto& columnOp : keyDesc.Columns) { + keyDesc.ColumnInfos.clear(); + keyDesc.ColumnInfos.reserve(keyDesc.Columns.size()); + for (auto& columnOp : keyDesc.Columns) { if (IsSystemColumn(columnOp.Column)) { continue; } - - if (IsSysTable() && columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { - SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::OperationNotSupported); + + if (IsSysTable() && columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { + SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::OperationNotSupported); return false; - } - + } + const auto* column = columns.FindPtr(columnOp.Column); - - if (!column) { - entry.Status = TResolve::EStatus::TypeCheckError; - keyDesc.Status = TKeyDesc::EStatus::TypeCheckFailed; - keyDesc.ColumnInfos.push_back({ - columnOp.Column, 0, 0, TKeyDesc::EStatus::NotExists - }); - continue; - } - - if (column->PType != columnOp.ExpectedType) { - entry.Status = TResolve::EStatus::TypeCheckError; - keyDesc.Status = TKeyDesc::EStatus::TypeCheckFailed; - keyDesc.ColumnInfos.push_back({ - columnOp.Column, column->PType, 0, TKeyDesc::EStatus::TypeCheckFailed - }); - continue; - } - - if (columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { - entry.Status = TResolve::EStatus::TypeCheckError; - keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; - keyDesc.ColumnInfos.push_back({ - columnOp.Column, column->PType, 0, TKeyDesc::EStatus::OperationNotSupported - }); - continue; - } - } - + + if (!column) { + entry.Status = TResolve::EStatus::TypeCheckError; + keyDesc.Status = TKeyDesc::EStatus::TypeCheckFailed; + keyDesc.ColumnInfos.push_back({ + columnOp.Column, 0, 0, TKeyDesc::EStatus::NotExists + }); + continue; + } + + if (column->PType != columnOp.ExpectedType) { + entry.Status = TResolve::EStatus::TypeCheckError; + keyDesc.Status = TKeyDesc::EStatus::TypeCheckFailed; + keyDesc.ColumnInfos.push_back({ + columnOp.Column, column->PType, 0, TKeyDesc::EStatus::TypeCheckFailed + }); + continue; + } + + if (columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { + entry.Status = TResolve::EStatus::TypeCheckError; + keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; + keyDesc.ColumnInfos.push_back({ + columnOp.Column, column->PType, 0, TKeyDesc::EStatus::OperationNotSupported + }); + continue; + } + } + return true; } @@ -1755,10 +1755,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target); if (!schema) { - return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); + return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); } - if (!CheckColumns(context, entry, schema->KeyColumnTypes, schema->Columns)) { + if (!CheckColumns(context, entry, schema->KeyColumnTypes, schema->Columns)) { return; } @@ -1774,35 +1774,35 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { keyDesc.Status = TKeyDesc::EStatus::Ok; } - void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = TResponseProps()) const { + void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = TResponseProps()) const { SBC_LOG_D("FillEntry for TResolve" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); - + << ": self# " << Owner->SelfId() + << ", cacheItem# " << ToString() + << ", entry# " << entry.ToString() + << ", props# " << props.ToString()); + TKeyDesc& keyDesc = *entry.KeyDescription; - if (props.IsSync && props.Partial) { - return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); - } - + if (props.IsSync && props.Partial) { + return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); + } + if (Status && Status == NKikimrScheme::StatusPathDoesNotExist) { - return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); + return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); } if (!Status || Status != NKikimrScheme::StatusSuccess) { - return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); + return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists); } - if (!keyDesc.TableId.SysViewInfo.empty()) { + if (!keyDesc.TableId.SysViewInfo.empty()) { if (Kind == TNavigate::KindPath) { auto split = SplitPath(Path); if (split.size() == 1) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); } } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); // Add all shards of the OLAP store @@ -1820,241 +1820,241 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } return; } - - return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); + + return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists); } if (!Created) { - return SetError(context, entry, TResolve::EStatus::NotMaterialized, TKeyDesc::EStatus::NotExists); + return SetError(context, entry, TResolve::EStatus::NotMaterialized, TKeyDesc::EStatus::NotExists); } - entry.Kind = TableKind; + entry.Kind = TableKind; entry.DomainInfo = DomainInfo; - if (!CheckColumns(context, entry, KeyColumnTypes, Columns)) { + if (!CheckColumns(context, entry, KeyColumnTypes, Columns)) { return; } // fill partition info - if (IsSysTable()) { - if (keyDesc.Range.Point) { - ui64 shard = 0; - bool ok = TSysTables::TLocksTable::ExtractKey( - keyDesc.Range.From, TSysTables::TLocksTable::EColumns::DataShard, shard - ); - if (ok) { - keyDesc.Partitions.push_back(TKeyDesc::TPartitionInfo(shard)); - } else { - keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; - ++context->Request->ErrorCount; - } - } + if (IsSysTable()) { + if (keyDesc.Range.Point) { + ui64 shard = 0; + bool ok = TSysTables::TLocksTable::ExtractKey( + keyDesc.Range.From, TSysTables::TLocksTable::EColumns::DataShard, shard + ); + if (ok) { + keyDesc.Partitions.push_back(TKeyDesc::TPartitionInfo(shard)); + } else { + keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; + ++context->Request->ErrorCount; + } + } } else if (OlapTableInfo) { // TODO: return proper partitioning info (KIKIMR-11069) for (ui64 columnShard : OlapTableInfo->Description.GetSharding().GetColumnShards()) { keyDesc.Partitions.push_back(TKeyDesc::TPartitionInfo(columnShard)); keyDesc.Partitions.back().Range = TKeyDesc::TPartitionRangeInfo(); } - } else { - if (Partitioning) { - FillRangePartitioning(keyDesc.Range, keyDesc.Partitions); - } - } - - if (keyDesc.Partitions.empty()) { - entry.Status = TResolve::EStatus::TypeCheckError; - keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; - } - + } else { + if (Partitioning) { + FillRangePartitioning(keyDesc.Range, keyDesc.Partitions); + } + } + + if (keyDesc.Partitions.empty()) { + entry.Status = TResolve::EStatus::TypeCheckError; + keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported; + } + // fill ACL info - keyDesc.SecurityObject = SecurityObject; - - if (keyDesc.Status != TKeyDesc::EStatus::Unknown) { - ++context->Request->ErrorCount; - return; - } - - entry.Status = TResolve::EStatus::OkData; - keyDesc.Status = TKeyDesc::EStatus::Ok; - } - - private: - // internal - TSchemeCache* Owner; - TSubscriber Subscriber; - bool Filled; - - mutable THashMap<TVariantContextPtr, TVector<TRequest>> InFlight; - - // common + keyDesc.SecurityObject = SecurityObject; + + if (keyDesc.Status != TKeyDesc::EStatus::Unknown) { + ++context->Request->ErrorCount; + return; + } + + entry.Status = TResolve::EStatus::OkData; + keyDesc.Status = TKeyDesc::EStatus::Ok; + } + + private: + // internal + TSchemeCache* Owner; + TSubscriber Subscriber; + bool Filled; + + mutable THashMap<TVariantContextPtr, TVector<TRequest>> InFlight; + + // common TMaybe<NKikimrScheme::EStatus> Status; - TNavigate::EKind Kind; - TResolve::EKind TableKind; - bool Created; - ui64 CreateStep; + TNavigate::EKind Kind; + TResolve::EKind TableKind; + bool Created; + ui64 CreateStep; TPathId PathId; TString Path; TSet<ui64> AbandonedSchemeShardsIds; - TIntrusivePtr<TSecurityObject> SecurityObject; + TIntrusivePtr<TSecurityObject> SecurityObject; NSchemeCache::TDomainInfo::TPtr DomainInfo; - THashMap<TString, TString> Attributes; - bool IsPrivatePath; + THashMap<TString, TString> Attributes; + bool IsPrivatePath; bool IsVirtual; - + // Used for Table and Index ui64 SchemaVersion; - // domain & path specific - TIntrusivePtr<TNavigate::TListNodeEntry> ListNodeEntry; + // domain & path specific + TIntrusivePtr<TNavigate::TListNodeEntry> ListNodeEntry; TIntrusivePtr<TNavigate::TDomainDescription> DomainDescription; - - // table specific - THashMap<ui32, TSysTables::TTableColumnInfo> Columns; - TVector<NScheme::TTypeId> KeyColumnTypes; + + // table specific + THashMap<ui32, TSysTables::TTableColumnInfo> Columns; + TVector<NScheme::TTypeId> KeyColumnTypes; THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; - TVector<TKeyDesc::TPartitionInfo> Partitioning; - + TVector<TKeyDesc::TPartitionInfo> Partitioning; + TIntrusivePtr<TNavigate::TDirEntryInfo> Self; - // RTMR specific - TIntrusivePtr<TNavigate::TRtmrVolumeInfo> RtmrVolumeInfo; - - // Kesus specific - TIntrusivePtr<TNavigate::TKesusInfo> KesusInfo; - - // Solomon specific - TIntrusivePtr<TNavigate::TSolomonVolumeInfo> SolomonVolumeInfo; - - // PQ specific - TIntrusivePtr<TNavigate::TPQGroupInfo> PQGroupInfo; - + // RTMR specific + TIntrusivePtr<TNavigate::TRtmrVolumeInfo> RtmrVolumeInfo; + + // Kesus specific + TIntrusivePtr<TNavigate::TKesusInfo> KesusInfo; + + // Solomon specific + TIntrusivePtr<TNavigate::TSolomonVolumeInfo> SolomonVolumeInfo; + + // PQ specific + TIntrusivePtr<TNavigate::TPQGroupInfo> PQGroupInfo; + // OlapStore specific TIntrusivePtr<TNavigate::TOlapStoreInfo> OlapStoreInfo; TIntrusivePtr<TNavigate::TOlapTableInfo> OlapTableInfo; - // CDC specific - TIntrusivePtr<TNavigate::TCdcStreamInfo> CdcStreamInfo; - + // CDC specific + TIntrusivePtr<TNavigate::TCdcStreamInfo> CdcStreamInfo; + // Sequence specific TIntrusivePtr<TNavigate::TSequenceInfo> SequenceInfo; - // Replication specific - TIntrusivePtr<TNavigate::TReplicationInfo> ReplicationInfo; - - }; // TCacheItem - - struct TMerger { - TCacheItem& operator()(TCacheItem& dst, TCacheItem&& src) { - return dst.Merge(std::move(src)); - } - }; - - struct TEvicter { - void operator()(TCacheItem& val) { - return val.OnEvict(); - } - }; - - enum class EPathType { - RegularPath, - SysPath, - SysLocksV1, - SysLocksV2, - }; - - static EPathType PathType(const TStringBuf path) { + // Replication specific + TIntrusivePtr<TNavigate::TReplicationInfo> ReplicationInfo; + + }; // TCacheItem + + struct TMerger { + TCacheItem& operator()(TCacheItem& dst, TCacheItem&& src) { + return dst.Merge(std::move(src)); + } + }; + + struct TEvicter { + void operator()(TCacheItem& val) { + return val.OnEvict(); + } + }; + + enum class EPathType { + RegularPath, + SysPath, + SysLocksV1, + SysLocksV2, + }; + + static EPathType PathType(const TStringBuf path) { if (path == "/sys"sv) { - return EPathType::SysPath; + return EPathType::SysPath; } else if (path == "/sys/locks"sv) { - return EPathType::SysLocksV1; + return EPathType::SysLocksV1; } else if (path == "/sys/locks2"sv) { - return EPathType::SysLocksV2; - } - - return EPathType::RegularPath; - } - - static EPathType PathType(const TPathId& pathId) { - if (pathId == TPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks)) { - return EPathType::SysLocksV1; - } else if (pathId == TPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2)) { - return EPathType::SysLocksV2; - } - - return EPathType::RegularPath; - } - - static TInstant Now() { - return TlsActivationContext->Now(); - } - - template <typename TPath> - TSubscriber CreateSubscriber(const TPath& path, const ui64 tabletId, const ui64 domainOwnerId) const { - SBC_LOG_T("Create subscriber" - << ": self# " << SelfId() - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId); - - const auto& domains = *AppData()->DomainsInfo; + return EPathType::SysLocksV2; + } + + return EPathType::RegularPath; + } + + static EPathType PathType(const TPathId& pathId) { + if (pathId == TPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks)) { + return EPathType::SysLocksV1; + } else if (pathId == TPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2)) { + return EPathType::SysLocksV2; + } + + return EPathType::RegularPath; + } + + static TInstant Now() { + return TlsActivationContext->Now(); + } + + template <typename TPath> + TSubscriber CreateSubscriber(const TPath& path, const ui64 tabletId, const ui64 domainOwnerId) const { + SBC_LOG_T("Create subscriber" + << ": self# " << SelfId() + << ", path# " << path + << ", domainOwnerId# " << domainOwnerId); + + const auto& domains = *AppData()->DomainsInfo; const ui32 domainId = domains.GetDomainUidByTabletId(tabletId); const ui32 boardSSId = domains.GetDomain(domainId).DefaultSchemeBoardGroup; - - return TSubscriber( - Register(CreateSchemeBoardSubscriber(SelfId(), path, boardSSId, domainOwnerId)), domainOwnerId, path - ); + + return TSubscriber( + Register(CreateSchemeBoardSubscriber(SelfId(), path, boardSSId, domainOwnerId)), domainOwnerId, path + ); + } + + TSubscriber CreateSubscriber(const TPathId& pathId, const ui64 domainOwnerId) const { + return CreateSubscriber(pathId, pathId.OwnerId, domainOwnerId); } - TSubscriber CreateSubscriber(const TPathId& pathId, const ui64 domainOwnerId) const { - return CreateSubscriber(pathId, pathId.OwnerId, domainOwnerId); - } - template <typename TContextPtr, typename TEntry, typename TPathExtractor, typename TTabletIdExtractor> void HandleEntry(TContextPtr context, TEntry& entry, size_t index, TPathExtractor pathExtractor, TTabletIdExtractor tabletIdExtractor) { auto path = pathExtractor(entry); TCacheItem* cacheItem = Cache.FindPtr(path); - + if (!cacheItem) { const EPathType pathType = PathType(path); switch (pathType) { case EPathType::RegularPath: { const ui64 tabletId = tabletIdExtractor(entry); - const ui32 domainId = AppData()->DomainsInfo->GetDomainUidByTabletId(tabletId); + const ui32 domainId = AppData()->DomainsInfo->GetDomainUidByTabletId(tabletId); if (tabletId == ui64(NSchemeShard::InvalidTabletId) || domainId == TDomainsInfo::BadDomainId) { - return SetRootUnknown(context.Get(), entry); - } - - ui64 domainOwnerId = context->Request->DomainOwnerId; - if (!domainOwnerId && context->Request->DatabaseName) { - TCacheItem* dbItem = Cache.FindPtr(CanonizePath(context->Request->DatabaseName)); - if (!dbItem) { - return SetRootUnknown(context.Get(), entry); - } - - auto status = dbItem->GetStatus(); + return SetRootUnknown(context.Get(), entry); + } + + ui64 domainOwnerId = context->Request->DomainOwnerId; + if (!domainOwnerId && context->Request->DatabaseName) { + TCacheItem* dbItem = Cache.FindPtr(CanonizePath(context->Request->DatabaseName)); + if (!dbItem) { + return SetRootUnknown(context.Get(), entry); + } + + auto status = dbItem->GetStatus(); if (status && status == NKikimrScheme::StatusPathDoesNotExist) { - return SetPathNotExist(context.Get(), entry); - } - + return SetPathNotExist(context.Get(), entry); + } + if (!status || status != NKikimrScheme::StatusSuccess) { - return SetLookupError(context.Get(), entry); - } - - Y_VERIFY(dbItem->GetDomainInfo()); - domainOwnerId = dbItem->GetDomainInfo()->ExtractSchemeShard(); - } - - if (!domainOwnerId) { - domainOwnerId = tabletId; - } - - cacheItem = &Cache.Upsert(path, TCacheItem(this, CreateSubscriber(path, tabletId, domainOwnerId), false)); + return SetLookupError(context.Get(), entry); + } + + Y_VERIFY(dbItem->GetDomainInfo()); + domainOwnerId = dbItem->GetDomainInfo()->ExtractSchemeShard(); + } + + if (!domainOwnerId) { + domainOwnerId = tabletId; + } + + cacheItem = &Cache.Upsert(path, TCacheItem(this, CreateSubscriber(path, tabletId, domainOwnerId), false)); break; - } + } case EPathType::SysPath: cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); cacheItem->FillAsSysPath(); @@ -2064,72 +2064,72 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true)); cacheItem->FillAsSysLocks(pathType == EPathType::SysLocksV2); break; - } - } - + } + } + Cache.Promote(path); - if (cacheItem->IsFilled() && !entry.SyncVersion) { - cacheItem->FillEntry(context.Get(), entry); - } - + if (cacheItem->IsFilled() && !entry.SyncVersion) { + cacheItem->FillEntry(context.Get(), entry); + } + if (entry.SyncVersion) { cacheItem->AddInFlight(context, index, true); - } + } if (!cacheItem->IsFilled()) { cacheItem->AddInFlight(context, index, false); } - } - - TCacheItem* SwapSubscriberAndUpsert(TCacheItem* byPath, const TPathId& notifyPathId, const TString& notifyPath) { - TSubscriber subscriber = CreateSubscriber(byPath->GetPathId(), byPath->GetSubcriber().DomainOwnerId); - byPath->SwapSubscriber(subscriber); - - TCacheItem newItem(this, subscriber, false); - byPath->MoveInFlightNavigateByPathRequests(newItem); - - Cache.DeleteIndex(notifyPath); - return &Cache.Upsert(notifyPath, notifyPathId, std::move(newItem)); - } - - TCacheItem* ResolveCacheItemCommon(const TPathId& notifyPathId, const TString& notifyPath) { + } + + TCacheItem* SwapSubscriberAndUpsert(TCacheItem* byPath, const TPathId& notifyPathId, const TString& notifyPath) { + TSubscriber subscriber = CreateSubscriber(byPath->GetPathId(), byPath->GetSubcriber().DomainOwnerId); + byPath->SwapSubscriber(subscriber); + + TCacheItem newItem(this, subscriber, false); + byPath->MoveInFlightNavigateByPathRequests(newItem); + + Cache.DeleteIndex(notifyPath); + return &Cache.Upsert(notifyPath, notifyPathId, std::move(newItem)); + } + + TCacheItem* ResolveCacheItemCommon(const TPathId& notifyPathId, const TString& notifyPath) { if (!notifyPathId) { return Cache.FindPtr(notifyPath); } - + if (!notifyPath) { return Cache.FindPtr(notifyPathId); } - + TCacheItem* byPath = Cache.FindPtr(notifyPath); TCacheItem* byPathId = Cache.FindPtr(notifyPathId); if (byPath == byPathId) { return byPathId; } - + if (!byPath) { - TSubscriber subscriber = CreateSubscriber(notifyPath, notifyPathId.OwnerId, byPathId->GetSubcriber().DomainOwnerId); + TSubscriber subscriber = CreateSubscriber(notifyPath, notifyPathId.OwnerId, byPathId->GetSubcriber().DomainOwnerId); return &Cache.Upsert(notifyPath, notifyPathId, TCacheItem(this, subscriber, false)); } - + TCacheItem* byPathByPathId = Cache.FindPtr(byPath->GetPathId()); - + if (!byPathByPathId) { return &Cache.Upsert(notifyPath, notifyPathId, TCacheItem(this, TSubscriber(), false)); } - + Y_VERIFY(byPath == byPathByPathId); if (!byPath->IsFilled() || byPath->GetPathId().OwnerId == notifyPathId.OwnerId) { if (byPath->GetPathId() < notifyPathId) { - return SwapSubscriberAndUpsert(byPath, notifyPathId, notifyPath); - } + return SwapSubscriberAndUpsert(byPath, notifyPathId, notifyPath); + } return byPathId; - } - + } + return nullptr; } @@ -2141,10 +2141,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TCacheItem* byPathId = Cache.FindPtr(notify.PathId); SBC_LOG_D("ResolveCacheItem" - << ": self# " << SelfId() - << ", notify# " << notify.ToString() - << ", by path# " << (byPath ? byPath->ToString() : "nullptr") - << ", by pathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + << ": self# " << SelfId() + << ", notify# " << notify.ToString() + << ", by path# " << (byPath ? byPath->ToString() : "nullptr") + << ", by pathId# " << (byPathId ? byPathId->ToString() : "nullptr")); TCacheItem* commonResolve = ResolveCacheItemCommon(notify.PathId, notify.Path); if (commonResolve) { @@ -2158,8 +2158,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Y_VERIFY(byPath); Y_VERIFY(byPathId != byPath); - if (!byPath->GetDomainId() && notifyDomainId) { - return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); + if (!byPath->GetDomainId() && notifyDomainId) { + return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } SBC_LOG_D("ResolveCacheItemForNotify: subdomain case" @@ -2169,58 +2169,58 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); - if (byPath->GetDomainId() != notifyDomainId && notifyDomainId) { + if (byPath->GetDomainId() != notifyDomainId && notifyDomainId) { SBC_LOG_D("ResolveCacheItemForNotify: recreation domain case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId + << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") + << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); if (byPath->GetPathId() < notify.PathId) { - return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); + return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } - + return byPathId; } if (byPath->GetPathId() == notifyDomainId) { //Update from TSS, GSS->TSS if (byPath->GetAbandonedSchemeShardIds().contains(notify.PathId.OwnerId)) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update is ignored, present GSS reverted implicitly that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update is ignored, present GSS reverted implicitly that TSS" + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); return byPathId; } - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update owerrides GSS by path" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); - return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); + SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update owerrides GSS by path" + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); + return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } if (byPath->GetDomainId() == notify.PathId) { //Update from GSS, TSS->GSS if (abandonedSchemeShardIds.contains(byPath->GetPathId().OwnerId)) { //GSS reverts TSS - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update owerrides TSS by path, GSS implicilty reverts that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); - return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); - } - - if (!notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS that removes TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); - return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); - } - - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update us ignored, TSS is prefered" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update owerrides TSS by path, GSS implicilty reverts that TSS" + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); + return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); + } + + if (!notifyDomainId) { + SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS that removes TSS" + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); + return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); + } + + SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update us ignored, TSS is prefered" + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); return byPathId; } @@ -2239,14 +2239,14 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return byPathId; } - if (!notifyDomainId) { + if (!notifyDomainId) { SBC_LOG_D("ResolveCacheItemForNotify: path has gone, update only by pathId" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); - return byPathId; - } - + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId); + return byPathId; + } + Y_FAIL_S("Unknown update"); } @@ -2263,126 +2263,126 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } template <typename TEvent> - void HandleNotify(TEvent& ev) { - const TResponseProps response = TResponseProps::FromEvent(ev); + void HandleNotify(TEvent& ev) { + const TResponseProps response = TResponseProps::FromEvent(ev); auto& notify = *ev->Get(); - SBC_LOG_D("HandleNotify" - << ": self# " << SelfId() - << ", notify# " << notify.ToString()); - + SBC_LOG_D("HandleNotify" + << ": self# " << SelfId() + << ", notify# " << notify.ToString()); + if (notify.Path && notify.PathId) { - Y_VERIFY(!response.IsSync); + Y_VERIFY(!response.IsSync); } TCacheItem* cacheItem = ResolveCacheItemForNotify(notify); - if (!cacheItem) { + if (!cacheItem) { SBC_LOG_W("HandleNotify doesn't find any cacheItem for Fill" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", isSync# " << response.IsSync); - return; - } - - cacheItem->Fill(notify); - - if (!response.IsSync && notify.PathId) { - Y_VERIFY_S(cacheItem->GetPathId() == notify.PathId, "Inconsistent path ids" - << ": cacheItem# " << cacheItem->ToString() - << ", notification# " << notify.ToString()); - } - - for (const auto& x : cacheItem->ProcessInFlight(response)) { + << ": self# " << SelfId() + << ", path# " << notify.Path + << ", pathId# " << notify.PathId + << ", isSync# " << response.IsSync); + return; + } + + cacheItem->Fill(notify); + + if (!response.IsSync && notify.PathId) { + Y_VERIFY_S(cacheItem->GetPathId() == notify.PathId, "Inconsistent path ids" + << ": cacheItem# " << cacheItem->ToString() + << ", notification# " << notify.ToString()); + } + + for (const auto& x : cacheItem->ProcessInFlight(response)) { if (auto* context = std::get_if<TNavigateContextPtr>(&x)) { - if (!context->Get()->WaitCounter) { - Complete(*context); - } + if (!context->Get()->WaitCounter) { + Complete(*context); + } } else if (auto* context = std::get_if<TResolveContextPtr>(&x)) { - if (!context->Get()->WaitCounter) { - Complete(*context); - } - } else { - Y_FAIL("unknown context type"); - } - } - } - - template <typename TEvent> - bool MaybeRunDbResolver(TEvent& ev) { - auto& request = ev->Get()->Request; - - if (request->DomainOwnerId) { - return false; - } - - if (!request->DatabaseName) { - return false; - } - - const auto db = SplitPath(request->DatabaseName); - if (db.empty()) { - return false; - } - - const TString databaseName = CanonizePath(db); - - TCacheItem* dbItem = Cache.FindPtr(databaseName); - if (dbItem) { - Cache.Promote(databaseName); - if (dbItem->IsFilled()) { - return false; - } - } - - auto it = Roots.find(db.front()); - if (it == Roots.end()) { - return false; - } - + if (!context->Get()->WaitCounter) { + Complete(*context); + } + } else { + Y_FAIL("unknown context type"); + } + } + } + + template <typename TEvent> + bool MaybeRunDbResolver(TEvent& ev) { + auto& request = ev->Get()->Request; + + if (request->DomainOwnerId) { + return false; + } + + if (!request->DatabaseName) { + return false; + } + + const auto db = SplitPath(request->DatabaseName); + if (db.empty()) { + return false; + } + + const TString databaseName = CanonizePath(db); + + TCacheItem* dbItem = Cache.FindPtr(databaseName); + if (dbItem) { + Cache.Promote(databaseName); + if (dbItem->IsFilled()) { + return false; + } + } + + auto it = Roots.find(db.front()); + if (it == Roots.end()) { + return false; + } + Register(CreateDbResolver(SelfId(), ev->Sender, THolder(request.Release()), it->second)); - return true; - } - - template <typename TContextPtr> - void MaybeComplete(TContextPtr context) { - Counters.StartRequest( - context->Request->ResultSet.size(), - context->WaitCounter, - Accumulate(context->Request->ResultSet, 0, [](ui64 sum, const auto& entry) { - return sum + ui64(entry.SyncVersion); - }) - ); - - if (!context->WaitCounter) { - Complete(context); - } - } - - template <typename TContextPtr> - void Complete(TContextPtr context) { - Counters.FinishRequest(Now() - context->CreatedAt); - Register(CreateAclChecker(context)); - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet" - << ": self# " << SelfId() + return true; + } + + template <typename TContextPtr> + void MaybeComplete(TContextPtr context) { + Counters.StartRequest( + context->Request->ResultSet.size(), + context->WaitCounter, + Accumulate(context->Request->ResultSet, 0, [](ui64 sum, const auto& entry) { + return sum + ui64(entry.SyncVersion); + }) + ); + + if (!context->WaitCounter) { + Complete(context); + } + } + + template <typename TContextPtr> + void Complete(TContextPtr context) { + Counters.FinishRequest(Now() - context->CreatedAt); + Register(CreateAclChecker(context)); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) { + SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet" + << ": self# " << SelfId() << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); - - if (MaybeRunDbResolver(ev)) { - return; - } - - TIntrusivePtr<TNavigateContext> context(new TNavigateContext(ev->Sender, ev->Get()->Request, Now())); + + if (MaybeRunDbResolver(ev)) { + return; + } + + TIntrusivePtr<TNavigateContext> context(new TNavigateContext(ev->Sender, ev->Get()->Request, Now())); for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { auto& entry = context->Request->ResultSet[i]; - + if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) { auto pathExtractor = [this](TNavigate::TEntry& entry) { - if (AppData()->FeatureFlags.GetEnableSystemViews() + if (AppData()->FeatureFlags.GetEnableSystemViews() && (entry.Operation == TNavigate::OpPath || entry.Operation == TNavigate::OpTable)) { NSysView::ISystemViewResolver::TSystemViewPath sysViewPath; @@ -2391,7 +2391,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return CanonizePath(sysViewPath.Parent); } } - + TString path = CanonizePath(entry.Path); return path ? path : TString("/"); }; @@ -2420,47 +2420,47 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { }; HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); - } - } - - MaybeComplete(context); - } - - void Handle(TEvTxProxySchemeCache::TEvResolveKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySet" - << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); - - if (MaybeRunDbResolver(ev)) { - return; - } - - TIntrusivePtr<TResolveContext> context(new TResolveContext(ev->Sender, ev->Get()->Request, Now())); - - auto pathExtractor = [](const TResolve::TEntry& entry) { - const TKeyDesc* keyDesc = entry.KeyDescription.Get(); - Y_VERIFY(keyDesc != nullptr); + } + } + + MaybeComplete(context); + } + + void Handle(TEvTxProxySchemeCache::TEvResolveKeySet::TPtr& ev) { + SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySet" + << ": self# " << SelfId() + << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + + if (MaybeRunDbResolver(ev)) { + return; + } + + TIntrusivePtr<TResolveContext> context(new TResolveContext(ev->Sender, ev->Get()->Request, Now())); + + auto pathExtractor = [](const TResolve::TEntry& entry) { + const TKeyDesc* keyDesc = entry.KeyDescription.Get(); + Y_VERIFY(keyDesc != nullptr); return TPathId(keyDesc->TableId.PathId); - }; - - auto tabletIdExtractor = [](const TResolve::TEntry& entry) { + }; + + auto tabletIdExtractor = [](const TResolve::TEntry& entry) { return entry.KeyDescription->TableId.PathId.OwnerId; - }; - + }; + for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) { auto& entry = context->Request->ResultSet[i]; HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor); } - MaybeComplete(context); - } - - void Handle(TEvTxProxySchemeCache::TEvInvalidateTable::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvInvalidateTable" - << ": self# " << SelfId()); - Send(ev->Sender, new TEvTxProxySchemeCache::TEvInvalidateTableResult(ev->Get()->Sender)); - } - + MaybeComplete(context); + } + + void Handle(TEvTxProxySchemeCache::TEvInvalidateTable::TPtr& ev) { + SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvInvalidateTable" + << ": self# " << SelfId()); + Send(ev->Sender, new TEvTxProxySchemeCache::TEvInvalidateTableResult(ev->Get()->Sender)); + } + TActorId EnsureWatchCache() { if (!WatchCache) { WatchCache = Register(new TWatchCache()); @@ -2480,114 +2480,114 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TActivationContext::Send(ev.Release()); } - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); - auto& record = *response->Record.MutableCacheResponse(); - - record.SetItemsTotalCount(Cache.Size()); - record.SetItemsByPathCount(Cache.GetPrimaryIndex().size()); - record.SetItemsByPathIdCount(Cache.GetSecondaryIndex().size()); - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - TCacheItem* desc = nullptr; - if (record.HasPath()) { - desc = Cache.FindPtr(record.GetPath()); - } else if (record.HasPathId()) { - desc = Cache.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); - } - - Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(desc ? desc->ToJsonString() : "{}"), 0, ev->Cookie); - } - - void ShrinkCache() { - if (Cache.Shrink()) { - Send(SelfId(), new TEvents::TEvWakeup()); - } else { - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); - } - } - - void PassAway() override { - auto evicter = [](const auto& index) { - for (auto it = index.begin(); it != index.end(); ++it) { - it->second->Value.OnEvict(); - } - }; - - evicter(Cache.GetPrimaryIndex()); - evicter(Cache.GetSecondaryIndex()); - + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); + auto& record = *response->Record.MutableCacheResponse(); + + record.SetItemsTotalCount(Cache.Size()); + record.SetItemsByPathCount(Cache.GetPrimaryIndex().size()); + record.SetItemsByPathIdCount(Cache.GetSecondaryIndex().size()); + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + TCacheItem* desc = nullptr; + if (record.HasPath()) { + desc = Cache.FindPtr(record.GetPath()); + } else if (record.HasPathId()) { + desc = Cache.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); + } + + Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(desc ? desc->ToJsonString() : "{}"), 0, ev->Cookie); + } + + void ShrinkCache() { + if (Cache.Shrink()) { + Send(SelfId(), new TEvents::TEvWakeup()); + } else { + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup()); + } + } + + void PassAway() override { + auto evicter = [](const auto& index) { + for (auto it = index.begin(); it != index.end(); ++it) { + it->second->Value.OnEvict(); + } + }; + + evicter(Cache.GetPrimaryIndex()); + evicter(Cache.GetSecondaryIndex()); + if (WatchCache) { Send(WatchCache, new TEvents::TEvPoison); WatchCache = { }; } - TMonitorableActor::PassAway(); - } - -public: + TMonitorableActor::PassAway(); + } + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::PROXY_SCHEME_CACHE; - } - - TSchemeCache(NSchemeCache::TSchemeCacheConfig* config) - : Counters(config->Counters) - , Cache(TDuration::Minutes(2), TThis::Now) + } + + TSchemeCache(NSchemeCache::TSchemeCacheConfig* config) + : Counters(config->Counters) + , Cache(TDuration::Minutes(2), TThis::Now) , SystemViewResolver(NSysView::CreateSystemViewResolver()) - { - for (const auto& root : config->Roots) { - Roots.emplace(root.Name, root.RootSchemeShard); - } - } - - void Bootstrap() { - TMonitorableActor::Bootstrap(); - - Become(&TThis::StateWork); - ShrinkCache(); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySet, Handle); - hFunc(TEvTxProxySchemeCache::TEvResolveKeySet, Handle); - hFunc(TEvTxProxySchemeCache::TEvInvalidateTable, Handle); - + { + for (const auto& root : config->Roots) { + Roots.emplace(root.Name, root.RootSchemeShard); + } + } + + void Bootstrap() { + TMonitorableActor::Bootstrap(); + + Become(&TThis::StateWork); + ShrinkCache(); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySet, Handle); + hFunc(TEvTxProxySchemeCache::TEvResolveKeySet, Handle); + hFunc(TEvTxProxySchemeCache::TEvInvalidateTable, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchPathId, Handle); hFunc(TEvTxProxySchemeCache::TEvWatchRemove, Handle); - hFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleNotify); - hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleNotify); - hFunc(TSchemeBoardEvents::TEvSyncResponse, HandleNotify); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); - - sFunc(TEvents::TEvWakeup, ShrinkCache); - sFunc(TEvents::TEvPoison, PassAway); - } - } - -private: - THashMap<TString, ui64> Roots; - TCounters Counters; - - TDoubleIndexedCache<TString, TPathId, TCacheItem, TMerger, TEvicter> Cache; + hFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleNotify); + hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleNotify); + hFunc(TSchemeBoardEvents::TEvSyncResponse, HandleNotify); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); + + sFunc(TEvents::TEvWakeup, ShrinkCache); + sFunc(TEvents::TEvPoison, PassAway); + } + } + +private: + THashMap<TString, ui64> Roots; + TCounters Counters; + + TDoubleIndexedCache<TString, TPathId, TCacheItem, TMerger, TEvicter> Cache; THolder<NSysView::ISystemViewResolver> SystemViewResolver; - + TActorId WatchCache; -}; // TSchemeCache - -} // NSchemeBoard - -IActor* CreateSchemeBoardSchemeCache(NSchemeCache::TSchemeCacheConfig* config) { - return new NSchemeBoard::TSchemeCache(config); -} - -} // NKikimr +}; // TSchemeCache + +} // NSchemeBoard + +IActor* CreateSchemeBoardSchemeCache(NSchemeCache::TSchemeCacheConfig* config) { + return new NSchemeBoard::TSchemeCache(config); +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/cache.h b/ydb/core/tx/scheme_board/cache.h index 0c475acca6a..f40899569d0 100644 --- a/ydb/core/tx/scheme_board/cache.h +++ b/ydb/core/tx/scheme_board/cache.h @@ -1,11 +1,11 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/tx/scheme_cache/scheme_cache.h> - -namespace NKikimr { - -IActor* CreateSchemeBoardSchemeCache(NSchemeCache::TSchemeCacheConfig* config); - -} // NKikimr + +namespace NKikimr { + +IActor* CreateSchemeBoardSchemeCache(NSchemeCache::TSchemeCacheConfig* config); + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/cache_ut.cpp b/ydb/core/tx/scheme_board/cache_ut.cpp index 5d7ac200e66..c27b3517b34 100644 --- a/ydb/core/tx/scheme_board/cache_ut.cpp +++ b/ydb/core/tx/scheme_board/cache_ut.cpp @@ -1,35 +1,35 @@ -#include "cache.h" -#include "ut_helpers.h" - +#include "cache.h" +#include "ut_helpers.h" + #include <ydb/core/base/counters.h> #include <ydb/core/base/tablet_types.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NKikimr { -namespace NSchemeBoard { - -using namespace NSchemeShardUT_Private; -using TConfig = NSchemeCache::TSchemeCacheConfig; -using TNavigate = NSchemeCache::TSchemeCacheNavigate; -using TResolve = NSchemeCache::TSchemeCacheRequest; - -class TCacheTest: public TTestWithSchemeshard { -public: - void SetUp() override { - TTestWithSchemeshard::SetUp(); - - TIntrusivePtr<TConfig> config = new TConfig(); - config->Counters = new NMonitoring::TDynamicCounters; + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NKikimr { +namespace NSchemeBoard { + +using namespace NSchemeShardUT_Private; +using TConfig = NSchemeCache::TSchemeCacheConfig; +using TNavigate = NSchemeCache::TSchemeCacheNavigate; +using TResolve = NSchemeCache::TSchemeCacheRequest; + +class TCacheTest: public TTestWithSchemeshard { +public: + void SetUp() override { + TTestWithSchemeshard::SetUp(); + + TIntrusivePtr<TConfig> config = new TConfig(); + config->Counters = new NMonitoring::TDynamicCounters; config->Roots.push_back(TConfig::TTagEntry(0, TTestTxConfig::SchemeShard, "Root")); - SchemeCache = Context->Register(CreateSchemeBoardSchemeCache(config.Get())); - Context->EnableScheduleForActor(SchemeCache, true); + SchemeCache = Context->Register(CreateSchemeBoardSchemeCache(config.Get())); + Context->EnableScheduleForActor(SchemeCache, true); TestAlterSubDomain(*Context, 1, "/", "StoragePools { " @@ -37,16 +37,16 @@ public: " Kind: \"pool-kind-1\" " "} " " Name: \"Root\" "); - } - - UNIT_TEST_SUITE(TCacheTest); - UNIT_TEST(Navigate); - UNIT_TEST(Attributes); - UNIT_TEST(List); - UNIT_TEST(Recreate); - UNIT_TEST(RacyRecreateAndSync); - UNIT_TEST(RacyCreateAndSync); - UNIT_TEST(CheckAccess); + } + + UNIT_TEST_SUITE(TCacheTest); + UNIT_TEST(Navigate); + UNIT_TEST(Attributes); + UNIT_TEST(List); + UNIT_TEST(Recreate); + UNIT_TEST(RacyRecreateAndSync); + UNIT_TEST(RacyCreateAndSync); + UNIT_TEST(CheckAccess); UNIT_TEST(CheckSystemViewAccess); UNIT_TEST(SystemView); UNIT_TEST(SysLocks); @@ -57,15 +57,15 @@ public: UNIT_TEST(MigrationUndo); UNIT_TEST(MigrationDeletedPathNavigate); UNIT_TEST(WatchRoot); - UNIT_TEST_SUITE_END(); - - void Navigate(); - void Attributes(); - void List(); - void Recreate(); - void RacyRecreateAndSync(); - void RacyCreateAndSync(); - void CheckAccess(); + UNIT_TEST_SUITE_END(); + + void Navigate(); + void Attributes(); + void List(); + void Recreate(); + void RacyRecreateAndSync(); + void RacyCreateAndSync(); + void CheckAccess(); void CheckSystemViewAccess(); void SystemView(); void SysLocks(); @@ -76,38 +76,38 @@ public: void MigrationUndo(); void MigrationDeletedPathNavigate(); void WatchRoot(); - -protected: + +protected: TNavigate::TEntry TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, - const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired); + const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired); + + TNavigate::TEntry TestNavigate(const TString& path, TNavigate::EStatus expectedStatus = TNavigate::EStatus::Ok, + const TString& sid = TString(), TNavigate::EOp op = TNavigate::EOp::OpPath, + bool showPrivatePath = false, bool redirectRequired = true, bool syncVersion = false); - TNavigate::TEntry TestNavigate(const TString& path, TNavigate::EStatus expectedStatus = TNavigate::EStatus::Ok, - const TString& sid = TString(), TNavigate::EOp op = TNavigate::EOp::OpPath, - bool showPrivatePath = false, bool redirectRequired = true, bool syncVersion = false); - TNavigate::TEntry TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, const TString& expectedPath, const TString& sid = TString(), - TNavigate::EOp op = TNavigate::EOp::OpPath, bool showPrivatePath = false); + TNavigate::EOp op = TNavigate::EOp::OpPath, bool showPrivatePath = false); TResolve::TEntry TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus = TResolve::EStatus::OkData, - const TString& sid = TString()); - + const TString& sid = TString()); + TActorId TestWatch(const TPathId& pathId, const TActorId& watcher = {}, ui64 key = 0); void TestWatchRemove(const TActorId& watcher, ui64 key = 0); - NSchemeCache::TDescribeResult::TCPtr ExpectWatchUpdated(const TActorId& watcher, const TString& expectedPath = {}); + NSchemeCache::TDescribeResult::TCPtr ExpectWatchUpdated(const TActorId& watcher, const TString& expectedPath = {}); TPathId ExpectWatchDeleted(const TActorId& watcher); void CreateAndMigrateWithoutDecision(ui64& txId); -private: +private: TActorId SchemeCache; - -}; // TCacheTest - -UNIT_TEST_SUITE_REGISTRATION(TCacheTest); - -void TCacheTest::Navigate() { - TestNavigate("/Root", TNavigate::EStatus::Ok); + +}; // TCacheTest + +UNIT_TEST_SUITE_REGISTRATION(TCacheTest); + +void TCacheTest::Navigate() { + TestNavigate("/Root", TNavigate::EStatus::Ok); ui64 txId = 100; @@ -117,182 +117,182 @@ void TCacheTest::Navigate() { auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA"); - TestNavigateByTableId(TTableId(1ull << 56, 1), TNavigate::EStatus::RootUnknown, ""); -} - -void TCacheTest::Attributes() { + TestNavigateByTableId(TTableId(1ull << 56, 1), TNavigate::EStatus::RootUnknown, ""); +} + +void TCacheTest::Attributes() { NKikimrSchemeOp::TAlterUserAttributes attrs; - auto& attr = *attrs.AddUserAttributes(); - attr.SetKey("key"); - attr.SetValue("value"); - - ui64 txId = 100; + auto& attr = *attrs.AddUserAttributes(); + attr.SetKey("key"); + attr.SetValue("value"); + + ui64 txId = 100; TestMkDir(*Context, ++txId, "/Root", "DirA", {NKikimrScheme::StatusAccepted}, attrs); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - - auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); - UNIT_ASSERT_VALUES_EQUAL(entry.Attributes.size(), 1); - UNIT_ASSERT(entry.Attributes.contains("key")); - UNIT_ASSERT_VALUES_EQUAL(entry.Attributes["key"], "value"); -} - -void TCacheTest::List() { - ui64 txId = 100; - - TestMkDir(*Context, ++txId, "/Root", "DirA"); - TestMkDir(*Context, ++txId, "/Root/DirA", "DirB"); - TestMkDir(*Context, ++txId, "/Root/DirA", "DirC"); + + auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(entry.Attributes.size(), 1); + UNIT_ASSERT(entry.Attributes.contains("key")); + UNIT_ASSERT_VALUES_EQUAL(entry.Attributes["key"], "value"); +} + +void TCacheTest::List() { + ui64 txId = 100; + + TestMkDir(*Context, ++txId, "/Root", "DirA"); + TestMkDir(*Context, ++txId, "/Root/DirA", "DirB"); + TestMkDir(*Context, ++txId, "/Root/DirA", "DirC"); TestWaitNotification(*Context, {txId - 2, txId - 1, txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - - { - auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); - UNIT_ASSERT(!entry.ListNodeEntry); - } - { - auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, TString(), TNavigate::OpList); - UNIT_ASSERT(entry.ListNodeEntry); - UNIT_ASSERT_VALUES_EQUAL(entry.ListNodeEntry->Children.size(), 2); - } -} - -void TCacheTest::Recreate() { + + { + auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); + UNIT_ASSERT(!entry.ListNodeEntry); + } + { + auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, TString(), TNavigate::OpList); + UNIT_ASSERT(entry.ListNodeEntry); + UNIT_ASSERT_VALUES_EQUAL(entry.ListNodeEntry->Children.size(), 2); + } +} + +void TCacheTest::Recreate() { const TActorId edge = Context->AllocateEdgeActor(); - ui64 txId = 100; - - TestMkDir(*Context, ++txId, "/Root", "DirA"); + ui64 txId = 100; + + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, "/Root/DirA"); - - TestRmDir(*Context, ++txId, "/Root", "DirA"); + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, "/Root/DirA"); + + TestRmDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); auto pathId = ev->Get()->PathId; TTableId tableId(pathId.OwnerId, pathId.LocalPathId); TestResolve(tableId, TResolve::EStatus::PathErrorNotExist); - - TestMkDir(*Context, ++txId, "/Root", "DirA"); + + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); -} - -void TCacheTest::RacyCreateAndSync() { - THolder<IEventHandle> delayedSyncRequest; - auto prevObserver = Context->SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TSchemeBoardEvents::EvSyncRequest: - delayedSyncRequest.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - TNavigate::TEntry entry; - entry.Path = SplitPath("/Root/DirA"); - entry.SyncVersion = true; - entry.Operation = TNavigate::OpPath; - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back(entry); - - const TActorId edge = Context->AllocateEdgeActor(); - Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); - - if (!delayedSyncRequest) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayedSyncRequest](IEventHandle&) -> bool { - return bool(delayedSyncRequest); - }); - Context->DispatchEvents(opts); - } - - Context->SetObserverFunc(prevObserver); - - ui64 txId = 100; - TestMkDir(*Context, ++txId, "/Root", "DirA"); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); +} + +void TCacheTest::RacyCreateAndSync() { + THolder<IEventHandle> delayedSyncRequest; + auto prevObserver = Context->SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TSchemeBoardEvents::EvSyncRequest: + delayedSyncRequest.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + TNavigate::TEntry entry; + entry.Path = SplitPath("/Root/DirA"); + entry.SyncVersion = true; + entry.Operation = TNavigate::OpPath; + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back(entry); + + const TActorId edge = Context->AllocateEdgeActor(); + Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); + + if (!delayedSyncRequest) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayedSyncRequest](IEventHandle&) -> bool { + return bool(delayedSyncRequest); + }); + Context->DispatchEvents(opts); + } + + Context->SetObserverFunc(prevObserver); + + ui64 txId = 100; + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - - Context->TTestBasicRuntime::Send(delayedSyncRequest.Release(), 0, true); - auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ErrorCount, 0); -} - -void TCacheTest::RacyRecreateAndSync() { - ui64 txId = 100; - - TestMkDir(*Context, ++txId, "/Root", "DirA"); + + Context->TTestBasicRuntime::Send(delayedSyncRequest.Release(), 0, true); + auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ErrorCount, 0); +} + +void TCacheTest::RacyRecreateAndSync() { + ui64 txId = 100; + + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, "", TNavigate::EOp::OpPath, false, true, true); - - TestRmDir(*Context, ++txId, "/Root", "DirA"); + TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, "", TNavigate::EOp::OpPath, false, true, true); + + TestRmDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - TestNavigate("/Root/DirA", TNavigate::EStatus::PathErrorUnknown, "", TNavigate::EOp::OpPath, false, true, true); - - THolder<IEventHandle> delayedSyncRequest; - auto prevObserver = Context->SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TSchemeBoardEvents::EvSyncRequest: - delayedSyncRequest.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - TNavigate::TEntry entry; - entry.Path = SplitPath("/Root/DirA"); - entry.SyncVersion = true; - entry.Operation = TNavigate::OpPath; - - auto request = MakeHolder<TNavigate>(); - request->ResultSet.push_back(entry); - - const TActorId edge = Context->AllocateEdgeActor(); - Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); - - if (!delayedSyncRequest) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayedSyncRequest](IEventHandle&) -> bool { - return bool(delayedSyncRequest); - }); - Context->DispatchEvents(opts); - } - - Context->SetObserverFunc(prevObserver); - - TestMkDir(*Context, ++txId, "/Root", "DirA"); + TestNavigate("/Root/DirA", TNavigate::EStatus::PathErrorUnknown, "", TNavigate::EOp::OpPath, false, true, true); + + THolder<IEventHandle> delayedSyncRequest; + auto prevObserver = Context->SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TSchemeBoardEvents::EvSyncRequest: + delayedSyncRequest.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + TNavigate::TEntry entry; + entry.Path = SplitPath("/Root/DirA"); + entry.SyncVersion = true; + entry.Operation = TNavigate::OpPath; + + auto request = MakeHolder<TNavigate>(); + request->ResultSet.push_back(entry); + + const TActorId edge = Context->AllocateEdgeActor(); + Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); + + if (!delayedSyncRequest) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayedSyncRequest](IEventHandle&) -> bool { + return bool(delayedSyncRequest); + }); + Context->DispatchEvents(opts); + } + + Context->SetObserverFunc(prevObserver); + + TestMkDir(*Context, ++txId, "/Root", "DirA"); TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - - Context->TTestBasicRuntime::Send(delayedSyncRequest.Release(), 0, true); - auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ErrorCount, 0); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ResultSet.at(0).Status, TNavigate::EStatus::Ok); -} - -void TCacheTest::CheckAccess() { - ui64 txId = 100; - TestMkDir(*Context, ++txId, "/Root", "DirA"); - TestModifyACL(*Context, ++txId, "/Root", "DirA", TString(), "user0@builtin"); - - auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); - - auto byPath = TestNavigate("/Root/DirA", TNavigate::EStatus::PathErrorUnknown, "user1@builtin"); - UNIT_ASSERT_VALUES_EQUAL(CanonizePath(byPath.Path), "/Root/DirA"); - UNIT_ASSERT_VALUES_EQUAL(byPath.TableId, TTableId()); - TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, "user0@builtin"); - - auto byTableId = TestNavigateByTableId(entry.TableId, TNavigate::EStatus::PathErrorUnknown, "", "user1@builtin"); - UNIT_ASSERT_VALUES_EQUAL(CanonizePath(byTableId.Path), ""); - UNIT_ASSERT_VALUES_EQUAL(byTableId.TableId, entry.TableId); - TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA", "user0@builtin"); -} - + + Context->TTestBasicRuntime::Send(delayedSyncRequest.Release(), 0, true); + auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ErrorCount, 0); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Request->ResultSet.at(0).Status, TNavigate::EStatus::Ok); +} + +void TCacheTest::CheckAccess() { + ui64 txId = 100; + TestMkDir(*Context, ++txId, "/Root", "DirA"); + TestModifyACL(*Context, ++txId, "/Root", "DirA", TString(), "user0@builtin"); + + auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok); + + auto byPath = TestNavigate("/Root/DirA", TNavigate::EStatus::PathErrorUnknown, "user1@builtin"); + UNIT_ASSERT_VALUES_EQUAL(CanonizePath(byPath.Path), "/Root/DirA"); + UNIT_ASSERT_VALUES_EQUAL(byPath.TableId, TTableId()); + TestNavigate("/Root/DirA", TNavigate::EStatus::Ok, "user0@builtin"); + + auto byTableId = TestNavigateByTableId(entry.TableId, TNavigate::EStatus::PathErrorUnknown, "", "user1@builtin"); + UNIT_ASSERT_VALUES_EQUAL(CanonizePath(byTableId.Path), ""); + UNIT_ASSERT_VALUES_EQUAL(byTableId.TableId, entry.TableId); + TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA", "user0@builtin"); +} + void TCacheTest::CheckSystemViewAccess() { ui64 txId = 100; TestCreateSubDomain(*Context, ++txId, "/Root", "Name: \"SubDomainA\""); @@ -316,15 +316,15 @@ void TCacheTest::CheckSystemViewAccess() { TestResolve(tableId, TResolve::EStatus::PathErrorNotExist, "user1@builtin"); } -void TCacheTest::SystemView() { +void TCacheTest::SystemView() { auto entry = TestNavigate("/Root/.sys/partition_stats", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable); auto tableId = entry.TableId; UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats"); TestNavigateByTableId(tableId, TNavigate::EStatus::Ok, "/Root/.sys/partition_stats", TString(), TNavigate::OpTable); -} - +} + void TCacheTest::SysLocks() { { auto entry = TestNavigate("/sys/locks", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true); @@ -350,7 +350,7 @@ void TCacheTest::TableSchemaVersion() { TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); { - auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); + auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 1); } @@ -361,51 +361,51 @@ void TCacheTest::TableSchemaVersion() { TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); { - auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); + auto entry = TestNavigate("/Root/Table1", TNavigate::EStatus::Ok); UNIT_ASSERT_VALUES_EQUAL(entry.TableId.SchemaVersion, 2); } } TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus, - const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired) + const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired) { - auto& entry = request->ResultSet.back(); + auto& entry = request->ResultSet.back(); entry.Operation = op; - entry.ShowPrivatePath = showPrivatePath; + entry.ShowPrivatePath = showPrivatePath; entry.RedirectRequired = redirectRequired; - - if (sid) { - request->UserToken = new NACLib::TUserToken(sid, {}); - } - + + if (sid) { + request->UserToken = new NACLib::TUserToken(sid, {}); + } + const TActorId edge = Context->AllocateEdgeActor(); - Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); - auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); - - const TNavigate::TEntry result = ev->Get()->Request->ResultSet[0]; - UNIT_ASSERT_VALUES_EQUAL(result.Status, expectedStatus); + Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0, 0, 0, true); + auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySetResult>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); + + const TNavigate::TEntry result = ev->Get()->Request->ResultSet[0]; + UNIT_ASSERT_VALUES_EQUAL(result.Status, expectedStatus); return result; -} - +} + TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus, - const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired, bool syncVersion) + const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired, bool syncVersion) { auto request = MakeHolder<TNavigate>(); request->ResultSet.push_back({}); auto& entry = request->ResultSet.back(); entry.Path = SplitPath(path); - entry.SyncVersion = syncVersion; + entry.SyncVersion = syncVersion; - auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, redirectRequired); + auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, redirectRequired); return result; } TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus, - const TString& expectedPath, const TString& sid, TNavigate::EOp op, bool showPrivatePath) + const TString& expectedPath, const TString& sid, TNavigate::EOp op, bool showPrivatePath) { auto request = MakeHolder<TNavigate>(); request->ResultSet.push_back({}); @@ -414,38 +414,38 @@ TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNa entry.TableId = tableId; entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, true); + auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, true); UNIT_ASSERT_VALUES_EQUAL(CanonizePath(result.Path), expectedPath); return result; } TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus, const TString& sid) { - auto request = MakeHolder<TResolve>(); - - auto keyDesc = MakeHolder<TKeyDesc>( + auto request = MakeHolder<TResolve>(); + + auto keyDesc = MakeHolder<TKeyDesc>( tableId, - TTableRange({}), - TKeyDesc::ERowOperation::Unknown, - TVector<NScheme::TTypeId>(), TVector<TKeyDesc::TColumnOp>() - ); + TTableRange({}), + TKeyDesc::ERowOperation::Unknown, + TVector<NScheme::TTypeId>(), TVector<TKeyDesc::TColumnOp>() + ); request->ResultSet.emplace_back(std::move(keyDesc)); - - if (sid) { - request->UserToken = new NACLib::TUserToken(sid, {}); - } - + + if (sid) { + request->UserToken = new NACLib::TUserToken(sid, {}); + } + const TActorId edge = Context->AllocateEdgeActor(); - Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvResolveKeySet(request.Release()), 0, 0, 0, true); - auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvResolveKeySetResult>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); - + Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvResolveKeySet(request.Release()), 0, 0, 0, true); + auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvResolveKeySetResult>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT(!ev->Get()->Request->ResultSet.empty()); + TResolve::TEntry result = std::move(ev->Get()->Request->ResultSet[0]); - UNIT_ASSERT_VALUES_EQUAL(result.Status, expectedStatus); - return result; -} - + UNIT_ASSERT_VALUES_EQUAL(result.Status, expectedStatus); + return result; +} + TActorId TCacheTest::TestWatch(const TPathId& pathId, const TActorId& watcher, ui64 key) { const TActorId edge = watcher ? watcher : Context->AllocateEdgeActor(); Context->Send(SchemeCache, edge, new TEvTxProxySchemeCache::TEvWatchPathId(pathId, key), 0, 0, 0, true); @@ -456,11 +456,11 @@ void TCacheTest::TestWatchRemove(const TActorId& watcher, ui64 key) { Context->Send(SchemeCache, watcher, new TEvTxProxySchemeCache::TEvWatchRemove(key), 0, 0, 0, true); } -NSchemeCache::TDescribeResult::TCPtr TCacheTest::ExpectWatchUpdated(const TActorId& watcher, const TString& expectedPath) { +NSchemeCache::TDescribeResult::TCPtr TCacheTest::ExpectWatchUpdated(const TActorId& watcher, const TString& expectedPath) { auto ev = Context->GrabEdgeEvent<TEvTxProxySchemeCache::TEvWatchNotifyUpdated>(watcher); - if (expectedPath) { - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Path, expectedPath); - } + if (expectedPath) { + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Path, expectedPath); + } return ev->Get()->Result; } @@ -895,7 +895,7 @@ void TCacheTest::WatchRoot() { auto watcher = TestWatch(TPathId(TTestTxConfig::SchemeShard, 1)); { - auto result = ExpectWatchUpdated(watcher, "/Root"); + auto result = ExpectWatchUpdated(watcher, "/Root"); UNIT_ASSERT_VALUES_EQUAL(result->GetStatus(), NKikimrScheme::StatusSuccess); UNIT_ASSERT_VALUES_EQUAL(result->GetPathDescription().GetSelf().GetPathId(), 1u); UNIT_ASSERT_VALUES_EQUAL(result->GetPathDescription().GetChildren().size(), 0u); @@ -921,7 +921,7 @@ void TCacheTest::WatchRoot() { TestWatch(TPathId(TTestTxConfig::SchemeShard, dirPathId), watcher); { - auto result = ExpectWatchUpdated(watcher, "/Root/DirA"); + auto result = ExpectWatchUpdated(watcher, "/Root/DirA"); UNIT_ASSERT_VALUES_EQUAL(result->GetStatus(), NKikimrScheme::StatusSuccess); UNIT_ASSERT_VALUES_EQUAL(result->GetPathDescription().GetSelf().GetPathId(), dirPathId); } @@ -948,35 +948,35 @@ void TCacheTest::WatchRoot() { SimulateSleep(*Context, TDuration::Seconds(1)); } -class TCacheTestWithDrops: public TCacheTest { -public: - TTestContext::TEventObserver ObserverFunc() override { - return [](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TSchemeBoardEvents::EvNotifyUpdate: - case TSchemeBoardEvents::EvNotifyDelete: - case TSchemeBoardEvents::EvSyncResponse: - return TTestContext::EEventAction::DROP; - } - - return TTestContext::EEventAction::PROCESS; - }; - } - - UNIT_TEST_SUITE(TCacheTestWithDrops); - UNIT_TEST(LookupErrorUponEviction); - UNIT_TEST_SUITE_END(); - - void LookupErrorUponEviction(); - -}; // TCacheTestWithDrops - -UNIT_TEST_SUITE_REGISTRATION(TCacheTestWithDrops); - -void TCacheTestWithDrops::LookupErrorUponEviction() { - TestNavigate("/Root/with_sync", TNavigate::EStatus::LookupError, "", TNavigate::EOp::OpPath, false, true, true); - TestNavigate("/Root/without_sync", TNavigate::EStatus::LookupError, "", TNavigate::EOp::OpPath, false, true, false); -} - -} // NSchemeBoard -} // NKikimr +class TCacheTestWithDrops: public TCacheTest { +public: + TTestContext::TEventObserver ObserverFunc() override { + return [](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TSchemeBoardEvents::EvNotifyUpdate: + case TSchemeBoardEvents::EvNotifyDelete: + case TSchemeBoardEvents::EvSyncResponse: + return TTestContext::EEventAction::DROP; + } + + return TTestContext::EEventAction::PROCESS; + }; + } + + UNIT_TEST_SUITE(TCacheTestWithDrops); + UNIT_TEST(LookupErrorUponEviction); + UNIT_TEST_SUITE_END(); + + void LookupErrorUponEviction(); + +}; // TCacheTestWithDrops + +UNIT_TEST_SUITE_REGISTRATION(TCacheTestWithDrops); + +void TCacheTestWithDrops::LookupErrorUponEviction() { + TestNavigate("/Root/with_sync", TNavigate::EStatus::LookupError, "", TNavigate::EOp::OpPath, false, true, true); + TestNavigate("/Root/without_sync", TNavigate::EStatus::LookupError, "", TNavigate::EOp::OpPath, false, true, false); +} + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/defs.h b/ydb/core/tx/scheme_board/defs.h index 0fde5188877..bbae7d3179a 100644 --- a/ydb/core/tx/scheme_board/defs.h +++ b/ydb/core/tx/scheme_board/defs.h @@ -1,4 +1,4 @@ -#pragma once - +#pragma once + #include <ydb/core/base/defs.h> #include <ydb/core/base/events.h> diff --git a/ydb/core/tx/scheme_board/double_indexed.h b/ydb/core/tx/scheme_board/double_indexed.h index 16b7ee733bc..99dad5aea40 100644 --- a/ydb/core/tx/scheme_board/double_indexed.h +++ b/ydb/core/tx/scheme_board/double_indexed.h @@ -1,409 +1,409 @@ -#pragma once - -#include <util/datetime/base.h> -#include <util/generic/hash.h> -#include <util/generic/list.h> - -namespace NKikimr { - -namespace NPrivate { - -// used in TDoubleIndexedMap -struct TDefaultBaseItem {}; - -// used in TDoubleIndexedCache -struct TTimestampedItem { - TInstant LastAccess; -}; - -template < - typename TPrimaryKey, - typename TSecondaryKey, - typename TValue, - typename TMergeFunc, - template <typename...> typename TPIndex = THashMap, - template <typename...> typename TSIndex = THashMap, - template <typename...> typename TValueContainer = TList, - typename TBaseItem = TDefaultBaseItem -> -class TDoubleIndexedMapImpl { - struct TItem; - -public: - using TStorage = TValueContainer<TItem>; - using TPrimaryIndex = TPIndex<TPrimaryKey, typename TStorage::iterator>; - using TSecondaryIndex = TSIndex<TSecondaryKey, typename TStorage::iterator>; - -private: - struct TItem: public TBaseItem { - TValue Value; - typename TPrimaryIndex::iterator PrimaryIndexIterator; - typename TSecondaryIndex::iterator SecondaryIndexIterator; - - explicit TItem( - const TValue& value, - typename TPrimaryIndex::iterator primaryIndexIterator, - typename TSecondaryIndex::iterator secondaryIndexIterator) - : Value(value) - , PrimaryIndexIterator(primaryIndexIterator) - , SecondaryIndexIterator(secondaryIndexIterator) - { - } - - explicit TItem( - TValue&& value, - typename TPrimaryIndex::iterator primaryIndexIterator, - typename TSecondaryIndex::iterator secondaryIndexIterator) noexcept - : Value(std::move(value)) - , PrimaryIndexIterator(primaryIndexIterator) - , SecondaryIndexIterator(secondaryIndexIterator) - { - } - }; - -protected: - template <typename TV> - void EmplaceInStorage(TV&& val) { - Storage.emplace_back(TItem(std::forward<TV>(val), PrimaryIndex.end(), SecondaryIndex.end())); - } - - template <typename TK, typename std::enable_if_t<std::is_same_v<std::decay_t<TK>, TPrimaryKey>, int> = 0> - auto EmplaceInIndex(TK&& key) { - return PrimaryIndex.emplace(std::forward<TK>(key), Storage.end()); - } - - template <typename TK, typename std::enable_if_t<std::is_same_v<std::decay_t<TK>, TSecondaryKey>, int> = 0> - auto EmplaceInIndex(TK&& key) { - return SecondaryIndex.emplace(std::forward<TK>(key), Storage.end()); - } - - static void CrossSetIterators(typename TStorage::iterator st, typename TPrimaryIndex::iterator ind) { - st->PrimaryIndexIterator = ind; - ind->second = st; - } - - static void CrossSetIterators(typename TStorage::iterator st, typename TSecondaryIndex::iterator ind) { - st->SecondaryIndexIterator = ind; - ind->second = st; - } - - static typename TPrimaryIndex::iterator& IndexIterator(typename TPrimaryIndex::iterator it) { - return it->second->PrimaryIndexIterator; - } - - static typename TSecondaryIndex::iterator& IndexIterator(typename TSecondaryIndex::iterator it) { - return it->second->SecondaryIndexIterator; - } - - static typename TSecondaryIndex::iterator AnotherIndexIterator(typename TPrimaryIndex::iterator it) { - return it->second->SecondaryIndexIterator; - } - - static typename TPrimaryIndex::iterator AnotherIndexIterator(typename TSecondaryIndex::iterator it) { - return it->second->PrimaryIndexIterator; - } - - TSecondaryIndex& AnotherIndex(typename TPrimaryIndex::iterator) { - return SecondaryIndex; - } - - TPrimaryIndex& AnotherIndex(typename TSecondaryIndex::iterator) { - return PrimaryIndex; - } - - template <typename TPi, typename TSi> - void DeleteIndex(const typename TPi::key_type& key, TPi& pIndex, TSi& sIndex) { - auto pIt = pIndex.find(key); - if (pIt == pIndex.end()) { - return; - } - +#pragma once + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/generic/list.h> + +namespace NKikimr { + +namespace NPrivate { + +// used in TDoubleIndexedMap +struct TDefaultBaseItem {}; + +// used in TDoubleIndexedCache +struct TTimestampedItem { + TInstant LastAccess; +}; + +template < + typename TPrimaryKey, + typename TSecondaryKey, + typename TValue, + typename TMergeFunc, + template <typename...> typename TPIndex = THashMap, + template <typename...> typename TSIndex = THashMap, + template <typename...> typename TValueContainer = TList, + typename TBaseItem = TDefaultBaseItem +> +class TDoubleIndexedMapImpl { + struct TItem; + +public: + using TStorage = TValueContainer<TItem>; + using TPrimaryIndex = TPIndex<TPrimaryKey, typename TStorage::iterator>; + using TSecondaryIndex = TSIndex<TSecondaryKey, typename TStorage::iterator>; + +private: + struct TItem: public TBaseItem { + TValue Value; + typename TPrimaryIndex::iterator PrimaryIndexIterator; + typename TSecondaryIndex::iterator SecondaryIndexIterator; + + explicit TItem( + const TValue& value, + typename TPrimaryIndex::iterator primaryIndexIterator, + typename TSecondaryIndex::iterator secondaryIndexIterator) + : Value(value) + , PrimaryIndexIterator(primaryIndexIterator) + , SecondaryIndexIterator(secondaryIndexIterator) + { + } + + explicit TItem( + TValue&& value, + typename TPrimaryIndex::iterator primaryIndexIterator, + typename TSecondaryIndex::iterator secondaryIndexIterator) noexcept + : Value(std::move(value)) + , PrimaryIndexIterator(primaryIndexIterator) + , SecondaryIndexIterator(secondaryIndexIterator) + { + } + }; + +protected: + template <typename TV> + void EmplaceInStorage(TV&& val) { + Storage.emplace_back(TItem(std::forward<TV>(val), PrimaryIndex.end(), SecondaryIndex.end())); + } + + template <typename TK, typename std::enable_if_t<std::is_same_v<std::decay_t<TK>, TPrimaryKey>, int> = 0> + auto EmplaceInIndex(TK&& key) { + return PrimaryIndex.emplace(std::forward<TK>(key), Storage.end()); + } + + template <typename TK, typename std::enable_if_t<std::is_same_v<std::decay_t<TK>, TSecondaryKey>, int> = 0> + auto EmplaceInIndex(TK&& key) { + return SecondaryIndex.emplace(std::forward<TK>(key), Storage.end()); + } + + static void CrossSetIterators(typename TStorage::iterator st, typename TPrimaryIndex::iterator ind) { + st->PrimaryIndexIterator = ind; + ind->second = st; + } + + static void CrossSetIterators(typename TStorage::iterator st, typename TSecondaryIndex::iterator ind) { + st->SecondaryIndexIterator = ind; + ind->second = st; + } + + static typename TPrimaryIndex::iterator& IndexIterator(typename TPrimaryIndex::iterator it) { + return it->second->PrimaryIndexIterator; + } + + static typename TSecondaryIndex::iterator& IndexIterator(typename TSecondaryIndex::iterator it) { + return it->second->SecondaryIndexIterator; + } + + static typename TSecondaryIndex::iterator AnotherIndexIterator(typename TPrimaryIndex::iterator it) { + return it->second->SecondaryIndexIterator; + } + + static typename TPrimaryIndex::iterator AnotherIndexIterator(typename TSecondaryIndex::iterator it) { + return it->second->PrimaryIndexIterator; + } + + TSecondaryIndex& AnotherIndex(typename TPrimaryIndex::iterator) { + return SecondaryIndex; + } + + TPrimaryIndex& AnotherIndex(typename TSecondaryIndex::iterator) { + return PrimaryIndex; + } + + template <typename TPi, typename TSi> + void DeleteIndex(const typename TPi::key_type& key, TPi& pIndex, TSi& sIndex) { + auto pIt = pIndex.find(key); + if (pIt == pIndex.end()) { + return; + } + // storage entry must be kept by other index, we clean only one - Y_VERIFY(AnotherIndexIterator(pIt) != sIndex.end()); - - IndexIterator(pIt) = pIndex.end(); - pIndex.erase(pIt); - } - - template <typename TPi, typename TSi> - void Erase(const typename TPi::key_type& key, TPi& pIndex, TSi& sIndex) { - auto pIt = pIndex.find(key); - if (pIt == pIndex.end()) { - return; - } - - auto sIt = AnotherIndexIterator(pIt); - if (sIt != sIndex.end()) { - sIndex.erase(sIt); - } - - Storage.erase(pIt->second); - pIndex.erase(pIt); - } - - template <typename TI> - static TValue* FindPtr(const typename TI::key_type& key, TI& index) { - auto it = index.find(key); - if (it == index.end()) { - return nullptr; - } - - return &it->second->Value; - } - -public: - explicit TDoubleIndexedMapImpl(const TMergeFunc& mf = TMergeFunc()) - : Merge(mf) - { - } - - template <typename TK, typename TV> - TValue& Upsert(TK&& key, TV&& val) { - auto it = EmplaceInIndex(std::forward<TK>(key)); - - if (it.second) { - EmplaceInStorage(std::forward<TV>(val)); - CrossSetIterators(std::prev(Storage.end()), it.first); - } else { - Merge(it.first->second->Value, std::forward<TV>(val)); - } - - return it.first->second->Value; - } - - template <typename TPk, typename TSk, typename TV> - TValue& Upsert(TPk&& pKey, TSk&& sKey, TV&& val) { - auto pIt = EmplaceInIndex(std::forward<TPk>(pKey)); - auto sIt = EmplaceInIndex(std::forward<TSk>(sKey)); - - if (pIt.second && sIt.second) { + Y_VERIFY(AnotherIndexIterator(pIt) != sIndex.end()); + + IndexIterator(pIt) = pIndex.end(); + pIndex.erase(pIt); + } + + template <typename TPi, typename TSi> + void Erase(const typename TPi::key_type& key, TPi& pIndex, TSi& sIndex) { + auto pIt = pIndex.find(key); + if (pIt == pIndex.end()) { + return; + } + + auto sIt = AnotherIndexIterator(pIt); + if (sIt != sIndex.end()) { + sIndex.erase(sIt); + } + + Storage.erase(pIt->second); + pIndex.erase(pIt); + } + + template <typename TI> + static TValue* FindPtr(const typename TI::key_type& key, TI& index) { + auto it = index.find(key); + if (it == index.end()) { + return nullptr; + } + + return &it->second->Value; + } + +public: + explicit TDoubleIndexedMapImpl(const TMergeFunc& mf = TMergeFunc()) + : Merge(mf) + { + } + + template <typename TK, typename TV> + TValue& Upsert(TK&& key, TV&& val) { + auto it = EmplaceInIndex(std::forward<TK>(key)); + + if (it.second) { + EmplaceInStorage(std::forward<TV>(val)); + CrossSetIterators(std::prev(Storage.end()), it.first); + } else { + Merge(it.first->second->Value, std::forward<TV>(val)); + } + + return it.first->second->Value; + } + + template <typename TPk, typename TSk, typename TV> + TValue& Upsert(TPk&& pKey, TSk&& sKey, TV&& val) { + auto pIt = EmplaceInIndex(std::forward<TPk>(pKey)); + auto sIt = EmplaceInIndex(std::forward<TSk>(sKey)); + + if (pIt.second && sIt.second) { // no pKey, no sKey, insert new element - EmplaceInStorage(std::forward<TV>(val)); - auto it = std::prev(Storage.end()); - CrossSetIterators(it, pIt.first); - CrossSetIterators(it, sIt.first); - } else if (!pIt.second && sIt.second) { + EmplaceInStorage(std::forward<TV>(val)); + auto it = std::prev(Storage.end()); + CrossSetIterators(it, pIt.first); + CrossSetIterators(it, sIt.first); + } else if (!pIt.second && sIt.second) { // has pKey, no sKey, merge val into exited pKey - Y_VERIFY(AnotherIndexIterator(pIt.first) == AnotherIndex(pIt.first).end()); - Merge(pIt.first->second->Value, std::forward<TV>(val)); - CrossSetIterators(pIt.first->second, sIt.first); - } else if (pIt.second && !sIt.second) { + Y_VERIFY(AnotherIndexIterator(pIt.first) == AnotherIndex(pIt.first).end()); + Merge(pIt.first->second->Value, std::forward<TV>(val)); + CrossSetIterators(pIt.first->second, sIt.first); + } else if (pIt.second && !sIt.second) { // no pKey, has sKey, merge val into exited sKey - Y_VERIFY(AnotherIndexIterator(sIt.first) == AnotherIndex(sIt.first).end()); - Merge(sIt.first->second->Value, std::forward<TV>(val)); - CrossSetIterators(sIt.first->second, pIt.first); - } else if (pIt.first->second != sIt.first->second) { + Y_VERIFY(AnotherIndexIterator(sIt.first) == AnotherIndex(sIt.first).end()); + Merge(sIt.first->second->Value, std::forward<TV>(val)); + CrossSetIterators(sIt.first->second, pIt.first); + } else if (pIt.first->second != sIt.first->second) { // has pKey, has sKey, pKey is not equal to sKey, merge val into merged pKey and sKey - Merge(pIt.first->second->Value, std::forward<TV>(Merge(sIt.first->second->Value, std::forward<TV>(val)))); - Storage.erase(sIt.first->second); - CrossSetIterators(pIt.first->second, sIt.first); - } else { + Merge(pIt.first->second->Value, std::forward<TV>(Merge(sIt.first->second->Value, std::forward<TV>(val)))); + Storage.erase(sIt.first->second); + CrossSetIterators(pIt.first->second, sIt.first); + } else { // has pKey, has sKey, pKey is equal to sKey, merge val into existed item; - Merge(pIt.first->second->Value, std::forward<TV>(val)); - } - - return pIt.first->second->Value; - } - - void DeleteIndex(const TPrimaryKey& key) { - DeleteIndex(key, PrimaryIndex, SecondaryIndex); - } - - void DeleteIndex(const TSecondaryKey& key) { - DeleteIndex(key, SecondaryIndex, PrimaryIndex); - } - - void Erase(const TPrimaryKey& key) { - Erase(key, PrimaryIndex, SecondaryIndex); - } - - void Erase(const TSecondaryKey& key) { - Erase(key, SecondaryIndex, PrimaryIndex); - } - - TValue* FindPtr(const TPrimaryKey& key) { - return FindPtr(key, PrimaryIndex); - } - - const TValue* FindPtr(const TPrimaryKey& key) const { - return FindPtr(key, PrimaryIndex); - } - - TValue* FindPtr(const TSecondaryKey& key) { - return FindPtr(key, SecondaryIndex); - } - - const TValue* FindPtr(const TSecondaryKey& key) const { - return FindPtr(key, SecondaryIndex); - } - - auto Size() const { - return StorageSize(); - } - - auto StorageSize() const { - return Storage.size(); - } - - const TPrimaryIndex& GetPrimaryIndex() const { - return PrimaryIndex; - } - - const TSecondaryIndex& GetSecondaryIndex() const { - return SecondaryIndex; - } - -protected: - TStorage Storage; - TPrimaryIndex PrimaryIndex; - TSecondaryIndex SecondaryIndex; - -private: - TMergeFunc Merge; - -}; // TDoubleIndexedMapImpl - -} // NPrivate - -template < - typename TPrimaryKey, - typename TSecondaryKey, - typename TValue, - typename TMergeFunc, - template <typename...> typename TPIndex = THashMap, - template <typename...> typename TSIndex = THashMap, - template <typename...> typename TValueContainer = TList -> -class TDoubleIndexedMap: public NPrivate::TDoubleIndexedMapImpl< - TPrimaryKey, - TSecondaryKey, - TValue, - TMergeFunc, - TPIndex, - TSIndex, - TValueContainer> { - -public: - explicit TDoubleIndexedMap(const TMergeFunc& mf = TMergeFunc()) - : NPrivate::TDoubleIndexedMapImpl< - TPrimaryKey, - TSecondaryKey, - TValue, - TMergeFunc, - TPIndex, - TSIndex, - TValueContainer>(mf) - { - } -}; - -template < - typename TPrimaryKey, - typename TSecondaryKey, - typename TValue, - typename TMergeFunc, - typename TEvictFunc, - template <typename...> typename TPIndex = THashMap, - template <typename...> typename TSIndex = THashMap, - template <typename...> typename TValueContainer = TList -> -class TDoubleIndexedCache: public NPrivate::TDoubleIndexedMapImpl< - TPrimaryKey, - TSecondaryKey, - TValue, - TMergeFunc, - TPIndex, - TSIndex, - TValueContainer, - NPrivate::TTimestampedItem> { - -public: - typedef TInstant (*TInstantGetter)(); - -private: - template <typename TI> - void Promote(const typename TI::key_type& key, TI& index) { - auto it = index.find(key); - if (it == index.end()) { - return; - } - - this->Storage.splice(this->Storage.end(), this->Storage, it->second); - auto st = std::prev(this->Storage.end()); - - if (st->PrimaryIndexIterator != this->PrimaryIndex.end()) { - this->CrossSetIterators(st, st->PrimaryIndexIterator); - } - - if (st->SecondaryIndexIterator != this->SecondaryIndex.end()) { - this->CrossSetIterators(st, st->SecondaryIndexIterator); - } - - st->LastAccess = Now(); - } - -public: - explicit TDoubleIndexedCache( - const TDuration keep, - TInstantGetter now, - const TMergeFunc& mf = TMergeFunc(), - const TEvictFunc& ef = TEvictFunc()) - : NPrivate::TDoubleIndexedMapImpl< - TPrimaryKey, - TSecondaryKey, - TValue, - TMergeFunc, - TPIndex, - TSIndex, - TValueContainer, - NPrivate::TTimestampedItem>(mf) - , Keep(keep) - , Now(now) - , OnEvict(ef) - { - } - - void Promote(const TPrimaryKey& key) { - Promote(key, this->PrimaryIndex); - } - - void Promote(const TSecondaryKey& key) { - Promote(key, this->SecondaryIndex); - } - - bool Shrink() { - if (this->Storage.empty()) { - return false; - } - - auto& item = this->Storage.front(); - if ((Now() - item.LastAccess) < Keep) { - return false; - } - - if (item.PrimaryIndexIterator != this->PrimaryIndex.end()) { - this->PrimaryIndex.erase(item.PrimaryIndexIterator); - } - if (item.SecondaryIndexIterator != this->SecondaryIndex.end()) { - this->SecondaryIndex.erase(item.SecondaryIndexIterator); - } - - OnEvict(item.Value); - this->Storage.pop_front(); - - return true; - } - -private: - TDuration Keep; - TInstantGetter Now; - TEvictFunc OnEvict; - -}; // TDoubleIndexedCache - -} // NKikimr + Merge(pIt.first->second->Value, std::forward<TV>(val)); + } + + return pIt.first->second->Value; + } + + void DeleteIndex(const TPrimaryKey& key) { + DeleteIndex(key, PrimaryIndex, SecondaryIndex); + } + + void DeleteIndex(const TSecondaryKey& key) { + DeleteIndex(key, SecondaryIndex, PrimaryIndex); + } + + void Erase(const TPrimaryKey& key) { + Erase(key, PrimaryIndex, SecondaryIndex); + } + + void Erase(const TSecondaryKey& key) { + Erase(key, SecondaryIndex, PrimaryIndex); + } + + TValue* FindPtr(const TPrimaryKey& key) { + return FindPtr(key, PrimaryIndex); + } + + const TValue* FindPtr(const TPrimaryKey& key) const { + return FindPtr(key, PrimaryIndex); + } + + TValue* FindPtr(const TSecondaryKey& key) { + return FindPtr(key, SecondaryIndex); + } + + const TValue* FindPtr(const TSecondaryKey& key) const { + return FindPtr(key, SecondaryIndex); + } + + auto Size() const { + return StorageSize(); + } + + auto StorageSize() const { + return Storage.size(); + } + + const TPrimaryIndex& GetPrimaryIndex() const { + return PrimaryIndex; + } + + const TSecondaryIndex& GetSecondaryIndex() const { + return SecondaryIndex; + } + +protected: + TStorage Storage; + TPrimaryIndex PrimaryIndex; + TSecondaryIndex SecondaryIndex; + +private: + TMergeFunc Merge; + +}; // TDoubleIndexedMapImpl + +} // NPrivate + +template < + typename TPrimaryKey, + typename TSecondaryKey, + typename TValue, + typename TMergeFunc, + template <typename...> typename TPIndex = THashMap, + template <typename...> typename TSIndex = THashMap, + template <typename...> typename TValueContainer = TList +> +class TDoubleIndexedMap: public NPrivate::TDoubleIndexedMapImpl< + TPrimaryKey, + TSecondaryKey, + TValue, + TMergeFunc, + TPIndex, + TSIndex, + TValueContainer> { + +public: + explicit TDoubleIndexedMap(const TMergeFunc& mf = TMergeFunc()) + : NPrivate::TDoubleIndexedMapImpl< + TPrimaryKey, + TSecondaryKey, + TValue, + TMergeFunc, + TPIndex, + TSIndex, + TValueContainer>(mf) + { + } +}; + +template < + typename TPrimaryKey, + typename TSecondaryKey, + typename TValue, + typename TMergeFunc, + typename TEvictFunc, + template <typename...> typename TPIndex = THashMap, + template <typename...> typename TSIndex = THashMap, + template <typename...> typename TValueContainer = TList +> +class TDoubleIndexedCache: public NPrivate::TDoubleIndexedMapImpl< + TPrimaryKey, + TSecondaryKey, + TValue, + TMergeFunc, + TPIndex, + TSIndex, + TValueContainer, + NPrivate::TTimestampedItem> { + +public: + typedef TInstant (*TInstantGetter)(); + +private: + template <typename TI> + void Promote(const typename TI::key_type& key, TI& index) { + auto it = index.find(key); + if (it == index.end()) { + return; + } + + this->Storage.splice(this->Storage.end(), this->Storage, it->second); + auto st = std::prev(this->Storage.end()); + + if (st->PrimaryIndexIterator != this->PrimaryIndex.end()) { + this->CrossSetIterators(st, st->PrimaryIndexIterator); + } + + if (st->SecondaryIndexIterator != this->SecondaryIndex.end()) { + this->CrossSetIterators(st, st->SecondaryIndexIterator); + } + + st->LastAccess = Now(); + } + +public: + explicit TDoubleIndexedCache( + const TDuration keep, + TInstantGetter now, + const TMergeFunc& mf = TMergeFunc(), + const TEvictFunc& ef = TEvictFunc()) + : NPrivate::TDoubleIndexedMapImpl< + TPrimaryKey, + TSecondaryKey, + TValue, + TMergeFunc, + TPIndex, + TSIndex, + TValueContainer, + NPrivate::TTimestampedItem>(mf) + , Keep(keep) + , Now(now) + , OnEvict(ef) + { + } + + void Promote(const TPrimaryKey& key) { + Promote(key, this->PrimaryIndex); + } + + void Promote(const TSecondaryKey& key) { + Promote(key, this->SecondaryIndex); + } + + bool Shrink() { + if (this->Storage.empty()) { + return false; + } + + auto& item = this->Storage.front(); + if ((Now() - item.LastAccess) < Keep) { + return false; + } + + if (item.PrimaryIndexIterator != this->PrimaryIndex.end()) { + this->PrimaryIndex.erase(item.PrimaryIndexIterator); + } + if (item.SecondaryIndexIterator != this->SecondaryIndex.end()) { + this->SecondaryIndex.erase(item.SecondaryIndexIterator); + } + + OnEvict(item.Value); + this->Storage.pop_front(); + + return true; + } + +private: + TDuration Keep; + TInstantGetter Now; + TEvictFunc OnEvict; + +}; // TDoubleIndexedCache + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/double_indexed_ut.cpp b/ydb/core/tx/scheme_board/double_indexed_ut.cpp index 58e885c627c..9d6727dd087 100644 --- a/ydb/core/tx/scheme_board/double_indexed_ut.cpp +++ b/ydb/core/tx/scheme_board/double_indexed_ut.cpp @@ -1,127 +1,127 @@ -#include "double_indexed.h" - +#include "double_indexed.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/string.h> -#include <util/string/vector.h> - -namespace NKikimr { - -Y_UNIT_TEST_SUITE(DoubleIndexedTests) { - using TPKey = TString; - using TSKey = ui32; - - struct TValue { - TString Payload; - - TValue& Merge(TValue&& x) { - Payload = JoinStrings(TVector<TString>{Payload, x.Payload}, ""); - return *this; - } - - bool operator==(const TValue& x) const { - return Payload == x.Payload; - } - }; - - struct TMerger { - TValue& operator()(TValue& dst, TValue&& src) { - return dst.Merge(std::move(src)); - } - }; - - Y_UNIT_TEST(TestUpsertBySingleKey) { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - TValue& hello = diMap.Upsert(TPKey("test"), TValue{"hello"}); - UNIT_ASSERT_STRINGS_EQUAL(hello.Payload, "hello"); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - - TValue& world = diMap.Upsert(TSKey(42), TValue{"world"}); - UNIT_ASSERT_STRINGS_EQUAL(world.Payload, "world"); - UNIT_ASSERT_VALUES_EQUAL(2, diMap.Size()); - } - - Y_UNIT_TEST(TestUpsertByBothKeys) { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - TValue& helloWorld = diMap.Upsert(TPKey("test"), TSKey(42), TValue{"hello, world"}); - UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world"); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - } - - Y_UNIT_TEST(TestMerge) { - TPKey pKey("test"); - TSKey sKey(42); - - { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - diMap.Upsert(pKey, TValue{"hello"}); - TValue& helloWorld = diMap.Upsert(pKey, sKey, TValue{", world"}); - - UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world"); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - } - - { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - diMap.Upsert(sKey, TValue{"world"}); - TValue& worldHello = diMap.Upsert(pKey, sKey, TValue{", hello"}); - - UNIT_ASSERT_STRINGS_EQUAL(worldHello.Payload, "world, hello"); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - } - - { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - diMap.Upsert(pKey, TValue{"hello"}); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - - diMap.Upsert(sKey, TValue{", world"}); - UNIT_ASSERT_VALUES_EQUAL(2, diMap.Size()); - - TValue& helloWorld = diMap.Upsert(pKey, sKey, TValue{"!"}); - UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world!"); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - } - } - - template <typename TKey> - void TestFindImpl(const TKey& key) { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - TValue& value = diMap.Upsert(key, TValue{"value"}); - TValue* valuePtr = diMap.FindPtr(key); - UNIT_ASSERT_UNEQUAL(nullptr, valuePtr); - UNIT_ASSERT_EQUAL(&value, valuePtr); - } - - Y_UNIT_TEST(TestFind) { - TestFindImpl(TPKey("test")); - TestFindImpl(TSKey(42)); - } - - template <typename TKey> - void TestEraseImpl(const TKey& key) { - TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; - - diMap.Upsert(key, TValue{"value"}); - UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); - - diMap.Erase(key); - UNIT_ASSERT_VALUES_EQUAL(0, diMap.Size()); - - TValue* valuePtr = diMap.FindPtr(key); - UNIT_ASSERT_EQUAL(nullptr, valuePtr); - } - - Y_UNIT_TEST(TestErase) { - TestEraseImpl(TPKey("test")); - TestEraseImpl(TSKey(42)); - } -} - -} // NKikimr + +#include <util/generic/string.h> +#include <util/string/vector.h> + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(DoubleIndexedTests) { + using TPKey = TString; + using TSKey = ui32; + + struct TValue { + TString Payload; + + TValue& Merge(TValue&& x) { + Payload = JoinStrings(TVector<TString>{Payload, x.Payload}, ""); + return *this; + } + + bool operator==(const TValue& x) const { + return Payload == x.Payload; + } + }; + + struct TMerger { + TValue& operator()(TValue& dst, TValue&& src) { + return dst.Merge(std::move(src)); + } + }; + + Y_UNIT_TEST(TestUpsertBySingleKey) { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + TValue& hello = diMap.Upsert(TPKey("test"), TValue{"hello"}); + UNIT_ASSERT_STRINGS_EQUAL(hello.Payload, "hello"); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + + TValue& world = diMap.Upsert(TSKey(42), TValue{"world"}); + UNIT_ASSERT_STRINGS_EQUAL(world.Payload, "world"); + UNIT_ASSERT_VALUES_EQUAL(2, diMap.Size()); + } + + Y_UNIT_TEST(TestUpsertByBothKeys) { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + TValue& helloWorld = diMap.Upsert(TPKey("test"), TSKey(42), TValue{"hello, world"}); + UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world"); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + } + + Y_UNIT_TEST(TestMerge) { + TPKey pKey("test"); + TSKey sKey(42); + + { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + diMap.Upsert(pKey, TValue{"hello"}); + TValue& helloWorld = diMap.Upsert(pKey, sKey, TValue{", world"}); + + UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world"); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + } + + { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + diMap.Upsert(sKey, TValue{"world"}); + TValue& worldHello = diMap.Upsert(pKey, sKey, TValue{", hello"}); + + UNIT_ASSERT_STRINGS_EQUAL(worldHello.Payload, "world, hello"); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + } + + { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + diMap.Upsert(pKey, TValue{"hello"}); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + + diMap.Upsert(sKey, TValue{", world"}); + UNIT_ASSERT_VALUES_EQUAL(2, diMap.Size()); + + TValue& helloWorld = diMap.Upsert(pKey, sKey, TValue{"!"}); + UNIT_ASSERT_STRINGS_EQUAL(helloWorld.Payload, "hello, world!"); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + } + } + + template <typename TKey> + void TestFindImpl(const TKey& key) { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + TValue& value = diMap.Upsert(key, TValue{"value"}); + TValue* valuePtr = diMap.FindPtr(key); + UNIT_ASSERT_UNEQUAL(nullptr, valuePtr); + UNIT_ASSERT_EQUAL(&value, valuePtr); + } + + Y_UNIT_TEST(TestFind) { + TestFindImpl(TPKey("test")); + TestFindImpl(TSKey(42)); + } + + template <typename TKey> + void TestEraseImpl(const TKey& key) { + TDoubleIndexedMap<TPKey, TSKey, TValue, TMerger> diMap; + + diMap.Upsert(key, TValue{"value"}); + UNIT_ASSERT_VALUES_EQUAL(1, diMap.Size()); + + diMap.Erase(key); + UNIT_ASSERT_VALUES_EQUAL(0, diMap.Size()); + + TValue* valuePtr = diMap.FindPtr(key); + UNIT_ASSERT_EQUAL(nullptr, valuePtr); + } + + Y_UNIT_TEST(TestErase) { + TestEraseImpl(TPKey("test")); + TestEraseImpl(TSKey(42)); + } +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/events.h b/ydb/core/tx/scheme_board/events.h index 82a8100296f..018ebbff7db 100644 --- a/ydb/core/tx/scheme_board/events.h +++ b/ydb/core/tx/scheme_board/events.h @@ -1,100 +1,100 @@ -#pragma once - -#include "defs.h" -#include "helpers.h" -#include "two_part_description.h" - +#pragma once + +#include "defs.h" +#include "helpers.h" +#include "two_part_description.h" + #include <ydb/core/base/pathid.h> #include <ydb/core/base/statestorage.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/protos/scheme_board.pb.h> - -#include <util/generic/string.h> -#include <util/string/builder.h> - -namespace NKikimr { - -struct TSchemeBoardEvents { + +#include <util/generic/string.h> +#include <util/string/builder.h> + +namespace NKikimr { + +struct TSchemeBoardEvents { using TDescribeSchemeResult = NKikimrScheme::TEvDescribeSchemeResult; - - enum EEv { - // populator events - EvRequestDescribe = EventSpaceBegin(TKikimrEvents::ES_SCHEME_BOARD), - EvDescribeResult, - EvRequestUpdate, - - // replica <--> populator events - EvHandshakeRequest = EvRequestDescribe + 512, // <- - EvHandshakeResponse, // -> - EvUpdate, // <- - EvUpdateAck, // -> - EvCommitRequest, // <- - EvCommitResponse, // -> - - // subscriber <--> replica events - EvSubscribe = EvRequestDescribe + 2 * 512, // -> - EvUnsubscribe, // -> - EvNotify, // <- - EvSyncVersionRequest, // -> - EvSyncVersionResponse, // <- - EvNotifyAck, // -> - - // cache <--> subscriber events - EvNotifyUpdate = EvRequestDescribe + 3 * 512, // <- - EvNotifyDelete, // <- - EvSyncRequest, // -> - EvSyncResponse, // -> - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_BOARD), "expect End < EventSpaceEnd(ES_SCHEME_BOARD)"); - - // populator events - struct TEvRequestDescribe: public TEventLocal<TEvRequestDescribe, EvRequestDescribe> { + + enum EEv { + // populator events + EvRequestDescribe = EventSpaceBegin(TKikimrEvents::ES_SCHEME_BOARD), + EvDescribeResult, + EvRequestUpdate, + + // replica <--> populator events + EvHandshakeRequest = EvRequestDescribe + 512, // <- + EvHandshakeResponse, // -> + EvUpdate, // <- + EvUpdateAck, // -> + EvCommitRequest, // <- + EvCommitResponse, // -> + + // subscriber <--> replica events + EvSubscribe = EvRequestDescribe + 2 * 512, // -> + EvUnsubscribe, // -> + EvNotify, // <- + EvSyncVersionRequest, // -> + EvSyncVersionResponse, // <- + EvNotifyAck, // -> + + // cache <--> subscriber events + EvNotifyUpdate = EvRequestDescribe + 3 * 512, // <- + EvNotifyDelete, // <- + EvSyncRequest, // -> + EvSyncResponse, // -> + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_BOARD), "expect End < EventSpaceEnd(ES_SCHEME_BOARD)"); + + // populator events + struct TEvRequestDescribe: public TEventLocal<TEvRequestDescribe, EvRequestDescribe> { const TPathId PathId; const TActorId Replica; - - TEvRequestDescribe() = default; - + + TEvRequestDescribe() = default; + explicit TEvRequestDescribe(const TPathId pathId, const TActorId& replica) : PathId(pathId) , Replica(replica) - { - } - }; - - struct TEvDescribeResult: public TEventLocal<TEvDescribeResult, EvDescribeResult> { + { + } + }; + + struct TEvDescribeResult: public TEventLocal<TEvDescribeResult, EvDescribeResult> { const bool Commit = false; const TLocalPathId DeletedPathBegin = 0; // The points are inclusive const TLocalPathId DeletedPathEnd = 0; // [DeletedPathBegin; DeletedPathEnd] const TLocalPathId MigratedPathId = InvalidLocalPathId; - NSchemeBoard::TTwoPartDescription Description; - - TEvDescribeResult() = default; - - explicit TEvDescribeResult(const bool commit) - : Commit(commit) - { - } - + NSchemeBoard::TTwoPartDescription Description; + + TEvDescribeResult() = default; + + explicit TEvDescribeResult(const bool commit) + : Commit(commit) + { + } + explicit TEvDescribeResult(TLocalPathId deletedPathBegin, TLocalPathId deletedPathEnd) - : Commit(false) + : Commit(false) , DeletedPathBegin(deletedPathBegin) , DeletedPathEnd(deletedPathEnd) - { - } - - explicit TEvDescribeResult( + { + } + + explicit TEvDescribeResult( TLocalPathId deletedPathBegin, TLocalPathId deletedPathEnd, - const NSchemeBoard::TTwoPartDescription& description) - : Commit(false) + const NSchemeBoard::TTwoPartDescription& description) + : Commit(false) , DeletedPathBegin(deletedPathBegin) , DeletedPathEnd(deletedPathEnd) - , Description(description) - { - } - + , Description(description) + { + } + explicit TEvDescribeResult( TLocalPathId deletedPathBegin, TLocalPathId deletedPathEnd, TLocalPathId migratedPathId) @@ -105,17 +105,17 @@ struct TSchemeBoardEvents { { } - bool HasDeletedLocalPathIds() const { + bool HasDeletedLocalPathIds() const { return DeletedPathBegin != 0; - } - - bool HasMigratedPath() const { + } + + bool HasMigratedPath() const { return MigratedPathId != InvalidLocalPathId; } - bool HasDescription() const { - return Description; - } + bool HasDescription() const { + return Description; + } TString ToString() const override { return TStringBuilder() << ToStringHeader() << " {" @@ -125,79 +125,79 @@ struct TSchemeBoardEvents { << " msg: " << Description.Record.ShortDebugString() << " }"; } - }; - - struct TEvRequestUpdate: public TEventLocal<TEvRequestUpdate, EvRequestUpdate> { - const TPathId PathId; - - TEvRequestUpdate() = default; - - explicit TEvRequestUpdate(const TPathId pathId) - : PathId(pathId) - { - } - }; - - // replica <--> populator events - struct TEvHandshakeRequest: public TEventPB<TEvHandshakeRequest, NKikimrSchemeBoard::TEvHandshake, EvHandshakeRequest> { - TEvHandshakeRequest() = default; - - explicit TEvHandshakeRequest(const ui64 owner, const ui64 generation) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - } - }; - - struct TEvHandshakeResponse: public TEventPB<TEvHandshakeResponse, NKikimrSchemeBoard::TEvHandshake, EvHandshakeResponse> { - TEvHandshakeResponse() = default; - - explicit TEvHandshakeResponse(const ui64 owner, const ui64 generation) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - } - }; - - struct TEvUpdate: public TEventPreSerializedPB<TEvUpdate, NKikimrSchemeBoard::TEvUpdate, EvUpdate> { - TEvUpdate() = default; - - TPathId GetPathId() const { - if (!Record.HasLocalPathId()) { - return TPathId(); - } - - return TPathId( - Record.HasPathOwnerId() ? Record.GetPathOwnerId() : Record.GetOwner(), - Record.GetLocalPathId() - ); - } - }; - - struct TEvUpdateBuilder: public TEvUpdate { - using TBase::Record; - - TEvUpdateBuilder() = default; - - explicit TEvUpdateBuilder(const ui64 owner, const ui64 generation) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - } - - explicit TEvUpdateBuilder(const ui64 owner, const ui64 generation, const TPathId& pathId) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - Record.SetIsDeletion(true); - } - - explicit TEvUpdateBuilder( - const ui64 owner, - const ui64 generation, - const TDescribeSchemeResult& describeSchemeResult, - const bool isDeletion = false - ) { - Record.SetOwner(owner); - Record.SetGeneration(generation); + }; + + struct TEvRequestUpdate: public TEventLocal<TEvRequestUpdate, EvRequestUpdate> { + const TPathId PathId; + + TEvRequestUpdate() = default; + + explicit TEvRequestUpdate(const TPathId pathId) + : PathId(pathId) + { + } + }; + + // replica <--> populator events + struct TEvHandshakeRequest: public TEventPB<TEvHandshakeRequest, NKikimrSchemeBoard::TEvHandshake, EvHandshakeRequest> { + TEvHandshakeRequest() = default; + + explicit TEvHandshakeRequest(const ui64 owner, const ui64 generation) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + } + }; + + struct TEvHandshakeResponse: public TEventPB<TEvHandshakeResponse, NKikimrSchemeBoard::TEvHandshake, EvHandshakeResponse> { + TEvHandshakeResponse() = default; + + explicit TEvHandshakeResponse(const ui64 owner, const ui64 generation) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + } + }; + + struct TEvUpdate: public TEventPreSerializedPB<TEvUpdate, NKikimrSchemeBoard::TEvUpdate, EvUpdate> { + TEvUpdate() = default; + + TPathId GetPathId() const { + if (!Record.HasLocalPathId()) { + return TPathId(); + } + + return TPathId( + Record.HasPathOwnerId() ? Record.GetPathOwnerId() : Record.GetOwner(), + Record.GetLocalPathId() + ); + } + }; + + struct TEvUpdateBuilder: public TEvUpdate { + using TBase::Record; + + TEvUpdateBuilder() = default; + + explicit TEvUpdateBuilder(const ui64 owner, const ui64 generation) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + } + + explicit TEvUpdateBuilder(const ui64 owner, const ui64 generation, const TPathId& pathId) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + Record.SetPathOwnerId(pathId.OwnerId); + Record.SetLocalPathId(pathId.LocalPathId); + Record.SetIsDeletion(true); + } + + explicit TEvUpdateBuilder( + const ui64 owner, + const ui64 generation, + const TDescribeSchemeResult& describeSchemeResult, + const bool isDeletion = false + ) { + Record.SetOwner(owner); + Record.SetGeneration(generation); if (describeSchemeResult.HasPath()) { Record.SetPath(describeSchemeResult.GetPath()); } @@ -205,285 +205,285 @@ struct TSchemeBoardEvents { && describeSchemeResult.GetPathDescription().HasSelf()) { Record.SetPathOwnerId(describeSchemeResult.GetPathDescription().GetSelf().GetSchemeshardId()); } - Record.SetLocalPathId(describeSchemeResult.GetPathId()); + Record.SetLocalPathId(describeSchemeResult.GetPathId()); if (describeSchemeResult.HasPathOwnerId()) { Record.SetPathOwnerId(describeSchemeResult.GetPathOwnerId()); } - Record.SetIsDeletion(isDeletion); - } - - void SetDescribeSchemeResult(TString preSerialized) { - PreSerializedData = NSchemeBoard::PreSerializedProtoField( - std::move(preSerialized), Record.kDescribeSchemeResultFieldNumber); - } - - void SetDescribeSchemeResult(TDescribeSchemeResult describeSchemeResult) { - *Record.MutableDescribeSchemeResult() = std::move(describeSchemeResult); - } - - void SetDescribeSchemeResult(TString preSerialized, TDescribeSchemeResult describeSchemeResult) { - SetDescribeSchemeResult(std::move(preSerialized)); - SetDescribeSchemeResult(std::move(describeSchemeResult)); - } - - void SetDescribeSchemeResult(NSchemeBoard::TTwoPartDescription twoPart) { - SetDescribeSchemeResult(std::move(twoPart.PreSerialized), std::move(twoPart.Record)); - } - }; - - struct TEvUpdateAck: public TEventPB<TEvUpdateAck, NKikimrSchemeBoard::TEvUpdateAck, EvUpdateAck> { - TEvUpdateAck() = default; - - explicit TEvUpdateAck( - const ui64 owner, - const ui64 generation, + Record.SetIsDeletion(isDeletion); + } + + void SetDescribeSchemeResult(TString preSerialized) { + PreSerializedData = NSchemeBoard::PreSerializedProtoField( + std::move(preSerialized), Record.kDescribeSchemeResultFieldNumber); + } + + void SetDescribeSchemeResult(TDescribeSchemeResult describeSchemeResult) { + *Record.MutableDescribeSchemeResult() = std::move(describeSchemeResult); + } + + void SetDescribeSchemeResult(TString preSerialized, TDescribeSchemeResult describeSchemeResult) { + SetDescribeSchemeResult(std::move(preSerialized)); + SetDescribeSchemeResult(std::move(describeSchemeResult)); + } + + void SetDescribeSchemeResult(NSchemeBoard::TTwoPartDescription twoPart) { + SetDescribeSchemeResult(std::move(twoPart.PreSerialized), std::move(twoPart.Record)); + } + }; + + struct TEvUpdateAck: public TEventPB<TEvUpdateAck, NKikimrSchemeBoard::TEvUpdateAck, EvUpdateAck> { + TEvUpdateAck() = default; + + explicit TEvUpdateAck( + const ui64 owner, + const ui64 generation, const TPathId pathId, - const ui64 version - ) { - Record.SetOwner(owner); - Record.SetGeneration(generation); + const ui64 version + ) { + Record.SetOwner(owner); + Record.SetGeneration(generation); Record.SetLocalPathId(pathId.LocalPathId); - Record.SetVersion(version); + Record.SetVersion(version); Record.SetPathOwnerId(pathId.OwnerId); - } + } TPathId GetPathId() const { return TPathId( Record.HasPathOwnerId() ? Record.GetPathOwnerId() : Record.GetOwner(), Record.GetLocalPathId() - ); - } - }; - - struct TEvCommitRequest: public TEventPB<TEvCommitRequest, NKikimrSchemeBoard::TEvCommitGeneration, EvCommitRequest> { - TEvCommitRequest() = default; - - explicit TEvCommitRequest(const ui64 owner, const ui64 generation) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - } - }; - - struct TEvCommitResponse: public TEventPB<TEvCommitResponse, NKikimrSchemeBoard::TEvCommitGeneration, EvCommitResponse> { - TEvCommitResponse() = default; - - explicit TEvCommitResponse(const ui64 owner, const ui64 generation) { - Record.SetOwner(owner); - Record.SetGeneration(generation); - } - }; - - // subscriber <--> replica events - struct TEvSubscribe: public TEventPB<TEvSubscribe, NKikimrSchemeBoard::TEvSubscribe, EvSubscribe> { - TEvSubscribe() = default; - - explicit TEvSubscribe(const TString& path, const ui64 domainOwnerId) { - Record.SetPath(path); - Record.SetDomainOwnerId(domainOwnerId); - FillCapabilities(Record); - } - - explicit TEvSubscribe(const TPathId& pathId, const ui64 domainOwnerId) { + ); + } + }; + + struct TEvCommitRequest: public TEventPB<TEvCommitRequest, NKikimrSchemeBoard::TEvCommitGeneration, EvCommitRequest> { + TEvCommitRequest() = default; + + explicit TEvCommitRequest(const ui64 owner, const ui64 generation) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + } + }; + + struct TEvCommitResponse: public TEventPB<TEvCommitResponse, NKikimrSchemeBoard::TEvCommitGeneration, EvCommitResponse> { + TEvCommitResponse() = default; + + explicit TEvCommitResponse(const ui64 owner, const ui64 generation) { + Record.SetOwner(owner); + Record.SetGeneration(generation); + } + }; + + // subscriber <--> replica events + struct TEvSubscribe: public TEventPB<TEvSubscribe, NKikimrSchemeBoard::TEvSubscribe, EvSubscribe> { + TEvSubscribe() = default; + + explicit TEvSubscribe(const TString& path, const ui64 domainOwnerId) { + Record.SetPath(path); + Record.SetDomainOwnerId(domainOwnerId); + FillCapabilities(Record); + } + + explicit TEvSubscribe(const TPathId& pathId, const ui64 domainOwnerId) { Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - Record.SetDomainOwnerId(domainOwnerId); - FillCapabilities(Record); - } - - static void FillCapabilities(NKikimrSchemeBoard::TEvSubscribe& record) { - record.MutableCapabilities()->SetAckNotifications(true); - } - }; - - struct TEvUnsubscribe: public TEventPB<TEvUnsubscribe, NKikimrSchemeBoard::TEvUnsubscribe, EvUnsubscribe> { - TEvUnsubscribe() = default; - - explicit TEvUnsubscribe(const TString& path) { - Record.SetPath(path); - } - - explicit TEvUnsubscribe(const TPathId& pathId) { + Record.SetLocalPathId(pathId.LocalPathId); + Record.SetDomainOwnerId(domainOwnerId); + FillCapabilities(Record); + } + + static void FillCapabilities(NKikimrSchemeBoard::TEvSubscribe& record) { + record.MutableCapabilities()->SetAckNotifications(true); + } + }; + + struct TEvUnsubscribe: public TEventPB<TEvUnsubscribe, NKikimrSchemeBoard::TEvUnsubscribe, EvUnsubscribe> { + TEvUnsubscribe() = default; + + explicit TEvUnsubscribe(const TString& path) { + Record.SetPath(path); + } + + explicit TEvUnsubscribe(const TPathId& pathId) { Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - } - }; - - struct TEvNotify: public TEventPreSerializedPB<TEvNotify, NKikimrSchemeBoard::TEvNotify, EvNotify> { - TEvNotify() = default; - }; - - struct TEvNotifyBuilder: public TEvNotify { - using TBase::Record; - - TEvNotifyBuilder() = default; - - explicit TEvNotifyBuilder(const TString& path, const bool isDeletion = false) { - Record.SetPath(path); - Record.SetIsDeletion(isDeletion); - } - - explicit TEvNotifyBuilder(const TPathId& pathId, const bool isDeletion = false) { + Record.SetLocalPathId(pathId.LocalPathId); + } + }; + + struct TEvNotify: public TEventPreSerializedPB<TEvNotify, NKikimrSchemeBoard::TEvNotify, EvNotify> { + TEvNotify() = default; + }; + + struct TEvNotifyBuilder: public TEvNotify { + using TBase::Record; + + TEvNotifyBuilder() = default; + + explicit TEvNotifyBuilder(const TString& path, const bool isDeletion = false) { + Record.SetPath(path); + Record.SetIsDeletion(isDeletion); + } + + explicit TEvNotifyBuilder(const TPathId& pathId, const bool isDeletion = false) { Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - Record.SetIsDeletion(isDeletion); - } - - explicit TEvNotifyBuilder( - const TString& path, - const TPathId& pathId, - const bool isDeletion = false - ) { - Record.SetPath(path); + Record.SetLocalPathId(pathId.LocalPathId); + Record.SetIsDeletion(isDeletion); + } + + explicit TEvNotifyBuilder( + const TString& path, + const TPathId& pathId, + const bool isDeletion = false + ) { + Record.SetPath(path); Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - Record.SetIsDeletion(isDeletion); - } - - void SetDescribeSchemeResult(const TString& preSerialized) { - PreSerializedData = NSchemeBoard::PreSerializedProtoField( - preSerialized, Record.kDescribeSchemeResultFieldNumber); - } - }; - - struct TEvNotifyAck: public TEventPB<TEvNotifyAck, NKikimrSchemeBoard::TEvNotifyAck, EvNotifyAck> { - TEvNotifyAck() = default; - - explicit TEvNotifyAck(ui64 version) { - Record.SetVersion(version); - } - }; - - struct TEvSyncVersionRequest: public TEventPB<TEvSyncVersionRequest, NKikimrSchemeBoard::TEvSyncVersionRequest, EvSyncVersionRequest> { - TEvSyncVersionRequest() = default; - - explicit TEvSyncVersionRequest(const TString& path) { - Record.SetPath(path); - } - - explicit TEvSyncVersionRequest(const TPathId& pathId) { + Record.SetLocalPathId(pathId.LocalPathId); + Record.SetIsDeletion(isDeletion); + } + + void SetDescribeSchemeResult(const TString& preSerialized) { + PreSerializedData = NSchemeBoard::PreSerializedProtoField( + preSerialized, Record.kDescribeSchemeResultFieldNumber); + } + }; + + struct TEvNotifyAck: public TEventPB<TEvNotifyAck, NKikimrSchemeBoard::TEvNotifyAck, EvNotifyAck> { + TEvNotifyAck() = default; + + explicit TEvNotifyAck(ui64 version) { + Record.SetVersion(version); + } + }; + + struct TEvSyncVersionRequest: public TEventPB<TEvSyncVersionRequest, NKikimrSchemeBoard::TEvSyncVersionRequest, EvSyncVersionRequest> { + TEvSyncVersionRequest() = default; + + explicit TEvSyncVersionRequest(const TString& path) { + Record.SetPath(path); + } + + explicit TEvSyncVersionRequest(const TPathId& pathId) { Record.SetPathOwnerId(pathId.OwnerId); - Record.SetLocalPathId(pathId.LocalPathId); - } - }; - - struct TEvSyncVersionResponse: public TEventPB<TEvSyncVersionResponse, NKikimrSchemeBoard::TEvSyncVersionResponse, EvSyncVersionResponse> { - TEvSyncVersionResponse() = default; - - explicit TEvSyncVersionResponse(const ui64 version, const bool partial = false) { - Record.SetVersion(version); - Record.SetPartial(partial); - } - }; - - // cache <--> subscriber events - struct TEvNotifyUpdate: public TEventLocal<TEvNotifyUpdate, EvNotifyUpdate> { - TString Path; - TPathId PathId; - TDescribeSchemeResult DescribeSchemeResult; - - TEvNotifyUpdate() = default; - - explicit TEvNotifyUpdate( - const TString& path, - TDescribeSchemeResult&& describeSchemeResult) - : Path(path) - , DescribeSchemeResult(std::move(describeSchemeResult)) - { - } - - explicit TEvNotifyUpdate( - const TPathId& pathId, - TDescribeSchemeResult&& describeSchemeResult) - : PathId(pathId) - , DescribeSchemeResult(std::move(describeSchemeResult)) - { - } - - explicit TEvNotifyUpdate( - const TString& path, - const TPathId& pathId, - TDescribeSchemeResult&& describeSchemeResult) - : Path(path) - , PathId(pathId) - , DescribeSchemeResult(std::move(describeSchemeResult)) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " {" - << " Path: " << Path - << " PathId: " << PathId - << " DescribeSchemeResult: " << DescribeSchemeResult.ShortDebugString() - << " }"; - } - }; - - struct TEvNotifyDelete: public TEventLocal<TEvNotifyDelete, EvNotifyDelete> { - TString Path; - TPathId PathId; - bool Strong; - - TEvNotifyDelete() = default; - - explicit TEvNotifyDelete(const TString& path, bool strong) - : Path(path) - , Strong(strong) - { - } - - explicit TEvNotifyDelete(const TPathId& pathId, bool strong) - : PathId(pathId) - , Strong(strong) - { - } - - explicit TEvNotifyDelete(const TString& path, const TPathId& pathId, bool strong) - : Path(path) - , PathId(pathId) - , Strong(strong) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " {" - << " Path: " << Path - << " PathId: " << PathId - << " Strong: " << Strong - << " }"; - } - }; - - struct TEvSyncRequest: public TEventLocal<TEvSyncRequest, EvSyncRequest> { - }; - - struct TEvSyncResponse: public TEventLocal<TEvSyncResponse, EvSyncResponse> { - TString Path; - TPathId PathId; - bool Partial; - - TEvSyncResponse() = default; - - explicit TEvSyncResponse(const TString& path, const bool partial = false) - : Path(path) - , Partial(partial) - { - } - - explicit TEvSyncResponse(const TPathId& pathId, const bool partial = false) - : PathId(pathId) - , Partial(partial) - { - } - - TString ToString() const override { - return TStringBuilder() << ToStringHeader() << " {" - << " Path: " << Path - << " PathId: " << PathId - << " Partial: " << Partial - << " }"; - } - }; - -}; // TSchemeBoardEvents - -} // NKikimr + Record.SetLocalPathId(pathId.LocalPathId); + } + }; + + struct TEvSyncVersionResponse: public TEventPB<TEvSyncVersionResponse, NKikimrSchemeBoard::TEvSyncVersionResponse, EvSyncVersionResponse> { + TEvSyncVersionResponse() = default; + + explicit TEvSyncVersionResponse(const ui64 version, const bool partial = false) { + Record.SetVersion(version); + Record.SetPartial(partial); + } + }; + + // cache <--> subscriber events + struct TEvNotifyUpdate: public TEventLocal<TEvNotifyUpdate, EvNotifyUpdate> { + TString Path; + TPathId PathId; + TDescribeSchemeResult DescribeSchemeResult; + + TEvNotifyUpdate() = default; + + explicit TEvNotifyUpdate( + const TString& path, + TDescribeSchemeResult&& describeSchemeResult) + : Path(path) + , DescribeSchemeResult(std::move(describeSchemeResult)) + { + } + + explicit TEvNotifyUpdate( + const TPathId& pathId, + TDescribeSchemeResult&& describeSchemeResult) + : PathId(pathId) + , DescribeSchemeResult(std::move(describeSchemeResult)) + { + } + + explicit TEvNotifyUpdate( + const TString& path, + const TPathId& pathId, + TDescribeSchemeResult&& describeSchemeResult) + : Path(path) + , PathId(pathId) + , DescribeSchemeResult(std::move(describeSchemeResult)) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " Path: " << Path + << " PathId: " << PathId + << " DescribeSchemeResult: " << DescribeSchemeResult.ShortDebugString() + << " }"; + } + }; + + struct TEvNotifyDelete: public TEventLocal<TEvNotifyDelete, EvNotifyDelete> { + TString Path; + TPathId PathId; + bool Strong; + + TEvNotifyDelete() = default; + + explicit TEvNotifyDelete(const TString& path, bool strong) + : Path(path) + , Strong(strong) + { + } + + explicit TEvNotifyDelete(const TPathId& pathId, bool strong) + : PathId(pathId) + , Strong(strong) + { + } + + explicit TEvNotifyDelete(const TString& path, const TPathId& pathId, bool strong) + : Path(path) + , PathId(pathId) + , Strong(strong) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " Path: " << Path + << " PathId: " << PathId + << " Strong: " << Strong + << " }"; + } + }; + + struct TEvSyncRequest: public TEventLocal<TEvSyncRequest, EvSyncRequest> { + }; + + struct TEvSyncResponse: public TEventLocal<TEvSyncResponse, EvSyncResponse> { + TString Path; + TPathId PathId; + bool Partial; + + TEvSyncResponse() = default; + + explicit TEvSyncResponse(const TString& path, const bool partial = false) + : Path(path) + , Partial(partial) + { + } + + explicit TEvSyncResponse(const TPathId& pathId, const bool partial = false) + : PathId(pathId) + , Partial(partial) + { + } + + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " Path: " << Path + << " PathId: " << PathId + << " Partial: " << Partial + << " }"; + } + }; + +}; // TSchemeBoardEvents + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/helpers.cpp b/ydb/core/tx/scheme_board/helpers.cpp index 7eb60f6ab98..3b53323ad3f 100644 --- a/ydb/core/tx/scheme_board/helpers.cpp +++ b/ydb/core/tx/scheme_board/helpers.cpp @@ -1,39 +1,39 @@ -#include "helpers.h" - +#include "helpers.h" + #include <google/protobuf/io/coded_stream.h> #include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/wire_format_lite.h> - + #include <library/cpp/actors/core/executor_thread.h> #include <library/cpp/actors/core/interconnect.h> - -#include <util/stream/format.h> - -namespace NKikimr { -namespace NSchemeBoard { - + +#include <util/stream/format.h> + +namespace NKikimr { +namespace NSchemeBoard { + TActorId MakeInterconnectProxyId(const ui32 nodeId) { return TActivationContext::InterconnectProxy(nodeId); -} - +} + ui64 GetPathVersion(const NKikimrScheme::TEvDescribeSchemeResult& record) { - if (!record.HasPathDescription()) { - return 0; - } - - const auto& pathDescription = record.GetPathDescription(); - if (!pathDescription.HasSelf()) { - return 0; - } - - const auto& self = pathDescription.GetSelf(); - if (!self.HasPathVersion()) { - return 0; - } - - return self.GetPathVersion(); -} - + if (!record.HasPathDescription()) { + return 0; + } + + const auto& pathDescription = record.GetPathDescription(); + if (!pathDescription.HasSelf()) { + return 0; + } + + const auto& self = pathDescription.GetSelf(); + if (!self.HasPathVersion()) { + return 0; + } + + return self.GetPathVersion(); +} + TPathId GetPathId(const NKikimrScheme::TEvDescribeSchemeResult &record) { if (record.HasPathId() && record.HasPathOwnerId()) { return TPathId(record.GetPathOwnerId(), record.GetPathId()); @@ -64,9 +64,9 @@ TDomainId GetDomainId(const NKikimrScheme::TEvDescribeSchemeResult &record) { return TDomainId(); } - const auto& domainKey = pathDescription.GetDomainDescription().GetDomainKey(); + const auto& domainKey = pathDescription.GetDomainDescription().GetDomainKey(); - return TDomainId(domainKey.GetSchemeShard(), domainKey.GetPathId()); + return TDomainId(domainKey.GetSchemeShard(), domainKey.GetPathId()); } TSet<ui64> GetAbandonedSchemeShardIds(const NKikimrScheme::TEvDescribeSchemeResult &record) { @@ -78,40 +78,40 @@ TSet<ui64> GetAbandonedSchemeShardIds(const NKikimrScheme::TEvDescribeSchemeResu return TSet<ui64>( pathDescription.GetAbandonedTenantsSchemeShards().begin(), pathDescription.GetAbandonedTenantsSchemeShards().end() - ); + ); } -TIntrusivePtr<TEventSerializedData> SerializeEvent(IEventBase* ev) { - TAllocChunkSerializer serializer; +TIntrusivePtr<TEventSerializedData> SerializeEvent(IEventBase* ev) { + TAllocChunkSerializer serializer; Y_VERIFY(ev->SerializeToArcadiaStream(&serializer)); - return serializer.Release(ev->IsExtendedFormat()); -} - + return serializer.Release(ev->IsExtendedFormat()); +} + void MultiSend(const TVector<const TActorId*>& recipients, const TActorId& sender, TAutoPtr<IEventBase> ev, ui32 flags, ui64 cookie) { - auto buffer = SerializeEvent(ev.Get()); + auto buffer = SerializeEvent(ev.Get()); for (const TActorId* recipient : recipients) { - TlsActivationContext->Send(new IEventHandle( - ev->Type(), flags, *recipient, sender, buffer, cookie - )); - } -} - -TString PreSerializedProtoField(TString data, int fieldNo) { - using CodedOutputStream = google::protobuf::io::CodedOutputStream; - using StringOutputStream = google::protobuf::io::StringOutputStream; - using WireFormat = google::protobuf::internal::WireFormatLite; - - TString key; - { - StringOutputStream stream(&key); - CodedOutputStream output(&stream); - WireFormat::WriteTag(fieldNo, WireFormat::WireType::WIRETYPE_LENGTH_DELIMITED, &output); - output.WriteVarint32(data.size()); - } - - data.prepend(key); - return data; -} - -} // NSchemeBoard -} // NKikimr + TlsActivationContext->Send(new IEventHandle( + ev->Type(), flags, *recipient, sender, buffer, cookie + )); + } +} + +TString PreSerializedProtoField(TString data, int fieldNo) { + using CodedOutputStream = google::protobuf::io::CodedOutputStream; + using StringOutputStream = google::protobuf::io::StringOutputStream; + using WireFormat = google::protobuf::internal::WireFormatLite; + + TString key; + { + StringOutputStream stream(&key); + CodedOutputStream output(&stream); + WireFormat::WriteTag(fieldNo, WireFormat::WireType::WIRETYPE_LENGTH_DELIMITED, &output); + output.WriteVarint32(data.size()); + } + + data.prepend(key); + return data; +} + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/helpers.h b/ydb/core/tx/scheme_board/helpers.h index 7117e3ccc35..3b07df3e065 100644 --- a/ydb/core/tx/scheme_board/helpers.h +++ b/ydb/core/tx/scheme_board/helpers.h @@ -1,55 +1,55 @@ -#pragma once - -#include "defs.h" +#pragma once + +#include "defs.h" #include <ydb/core/base/pathid.h> - + #include <ydb/core/protos/flat_tx_scheme.pb.h> - + #include <library/cpp/actors/core/log.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/generic/set.h> - -#if defined SB_LOG_T || \ - defined SB_LOG_D || \ - defined SB_LOG_I || \ - defined SB_LOG_N || \ - defined SB_LOG_W || \ - defined SB_LOG_E || \ - defined SB_LOG_CRIT -#error log macro redefinition -#endif - -#define SB_LOG_T(service, stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_D(service, stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_I(service, stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_N(service, stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_W(service, stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_E(service, stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) + +#if defined SB_LOG_T || \ + defined SB_LOG_D || \ + defined SB_LOG_I || \ + defined SB_LOG_N || \ + defined SB_LOG_W || \ + defined SB_LOG_E || \ + defined SB_LOG_CRIT +#error log macro redefinition +#endif + +#define SB_LOG_T(service, stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) +#define SB_LOG_D(service, stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) +#define SB_LOG_I(service, stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) +#define SB_LOG_N(service, stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) +#define SB_LOG_W(service, stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) +#define SB_LOG_E(service, stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) #define SB_LOG_CRIT(service, stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) - -namespace NKikimr { -namespace NSchemeBoard { - + +namespace NKikimr { +namespace NSchemeBoard { + using TDomainId = TPathId; - + TActorId MakeInterconnectProxyId(const ui32 nodeId); ui64 GetPathVersion(const NKikimrScheme::TEvDescribeSchemeResult& record); TPathId GetPathId(const NKikimrScheme::TEvDescribeSchemeResult& record); TDomainId GetDomainId(const NKikimrScheme::TEvDescribeSchemeResult& record); TSet<ui64> GetAbandonedSchemeShardIds(const NKikimrScheme::TEvDescribeSchemeResult &record); - -TIntrusivePtr<TEventSerializedData> SerializeEvent(IEventBase* ev); - + +TIntrusivePtr<TEventSerializedData> SerializeEvent(IEventBase* ev); + void MultiSend(const TVector<const TActorId*>& recipients, const TActorId& sender, TAutoPtr<IEventBase> ev, ui32 flags = 0, ui64 cookie = 0); - -template <typename TEvent> + +template <typename TEvent> void MultiSend(const TVector<const TActorId*>& recipients, const TActorId& sender, THolder<TEvent> ev, ui32 flags = 0, ui64 cookie = 0) { - MultiSend(recipients, sender, static_cast<IEventBase*>(ev.Release()), flags, cookie); -} - -// Only string or embed message fields are supported (it used wire type = 2 internally) -TString PreSerializedProtoField(TString data, int fieldNo); - -} // NSchemeBoard -} // NKikimr + MultiSend(recipients, sender, static_cast<IEventBase*>(ev.Release()), flags, cookie); +} + +// Only string or embed message fields are supported (it used wire type = 2 internally) +TString PreSerializedProtoField(TString data, int fieldNo); + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/load_test.cpp b/ydb/core/tx/scheme_board/load_test.cpp index bf446b28f57..d993f1e5ed2 100644 --- a/ydb/core/tx/scheme_board/load_test.cpp +++ b/ydb/core/tx/scheme_board/load_test.cpp @@ -1,424 +1,424 @@ -#include "events.h" -#include "helpers.h" -#include "load_test.h" -#include "populator.h" -#include "subscriber.h" - +#include "events.h" +#include "helpers.h" +#include "load_test.h" +#include "populator.h" +#include "subscriber.h" + #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/base/tabletid.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/ptr.h> -#include <util/generic/ylimits.h> -#include <util/random/random.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NSchemeBoard { - -namespace { - + +#include <util/generic/ptr.h> +#include <util/generic/ylimits.h> +#include <util/random/random.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NSchemeBoard { + +namespace { + bool IsDir(const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& self = record.GetPathDescription().GetSelf(); + const auto& self = record.GetPathDescription().GetSelf(); return self.GetParentPathId() == NSchemeShard::RootPathId; - } - - bool IsDir(const TTwoPartDescription& desc) { - return IsDir(desc.Record); - } - -} // anonymous - -class TLoadProducer: public TActorBootstrapped<TLoadProducer> { + } + + bool IsDir(const TTwoPartDescription& desc) { + return IsDir(desc.Record); + } + +} // anonymous + +class TLoadProducer: public TActorBootstrapped<TLoadProducer> { using TDescription = NKikimrScheme::TEvDescribeSchemeResult; - using TDescriptions = TMap<TPathId, TTwoPartDescription>; + using TDescriptions = TMap<TPathId, TTwoPartDescription>; using TDescribeSchemeResult = NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder; - - enum EWakeupTag { - TAG_MODIFY, - TAG_CHANGE, - }; - - static TDescriptions GenerateDescriptions(ui64 owner, const TTestConfig& config, ui64& nextPathId) { - const ui64 version = TlsActivationContext->Now().GetValue(); - - TDescriptions descriptions; - - for (ui64 dir = 1; dir <= config.Dirs; ++dir) { - const TString dirName = TStringBuilder() << "dir-" << dir; - const TString dirPath = TStringBuilder() << "/Root" << "/" << dirName; - - TTwoPartDescription dirDescTwoPart; - TDescription& dirDesc = dirDescTwoPart.Record; - + + enum EWakeupTag { + TAG_MODIFY, + TAG_CHANGE, + }; + + static TDescriptions GenerateDescriptions(ui64 owner, const TTestConfig& config, ui64& nextPathId) { + const ui64 version = TlsActivationContext->Now().GetValue(); + + TDescriptions descriptions; + + for (ui64 dir = 1; dir <= config.Dirs; ++dir) { + const TString dirName = TStringBuilder() << "dir-" << dir; + const TString dirPath = TStringBuilder() << "/Root" << "/" << dirName; + + TTwoPartDescription dirDescTwoPart; + TDescription& dirDesc = dirDescTwoPart.Record; + dirDesc.SetStatus(NKikimrScheme::StatusSuccess); - dirDesc.SetPathOwner(owner); - dirDesc.SetPathId(nextPathId++); - dirDesc.SetPath(dirPath); - - auto& dirSelf = *dirDesc.MutablePathDescription()->MutableSelf(); - dirSelf.SetName(dirName); - dirSelf.SetPathId(dirDesc.GetPathId()); - dirSelf.SetSchemeshardId(dirDesc.GetPathOwner()); + dirDesc.SetPathOwner(owner); + dirDesc.SetPathId(nextPathId++); + dirDesc.SetPath(dirPath); + + auto& dirSelf = *dirDesc.MutablePathDescription()->MutableSelf(); + dirSelf.SetName(dirName); + dirSelf.SetPathId(dirDesc.GetPathId()); + dirSelf.SetSchemeshardId(dirDesc.GetPathOwner()); dirSelf.SetPathType(NKikimrSchemeOp::EPathTypeDir); - dirSelf.SetCreateFinished(true); - dirSelf.SetCreateTxId(1); - dirSelf.SetCreateStep(1); + dirSelf.SetCreateFinished(true); + dirSelf.SetCreateTxId(1); + dirSelf.SetCreateStep(1); dirSelf.SetParentPathId(NSchemeShard::RootPathId); dirSelf.SetPathState(NKikimrSchemeOp::EPathStateNoChanges); - dirSelf.SetPathVersion(version); - - auto& dirChildren = *dirDesc.MutablePathDescription()->MutableChildren(); - dirChildren.Reserve(config.ObjectsPerDir); - - for (ui64 obj = 1; obj <= config.ObjectsPerDir; ++obj) { - const TString objName = TStringBuilder() << "obj-" << obj; - const TString objPath = TStringBuilder() << dirPath << "/" << objName; - - TTwoPartDescription objDescTwoPart; - TDescription& objDesc = objDescTwoPart.Record; - + dirSelf.SetPathVersion(version); + + auto& dirChildren = *dirDesc.MutablePathDescription()->MutableChildren(); + dirChildren.Reserve(config.ObjectsPerDir); + + for (ui64 obj = 1; obj <= config.ObjectsPerDir; ++obj) { + const TString objName = TStringBuilder() << "obj-" << obj; + const TString objPath = TStringBuilder() << dirPath << "/" << objName; + + TTwoPartDescription objDescTwoPart; + TDescription& objDesc = objDescTwoPart.Record; + objDesc.SetStatus(NKikimrScheme::StatusSuccess); - objDesc.SetPathOwner(owner); - objDesc.SetPathId(nextPathId++); - objDesc.SetPath(objPath); - - auto& objSelf = *objDesc.MutablePathDescription()->MutableSelf(); - objSelf.SetName(objName); + objDesc.SetPathOwner(owner); + objDesc.SetPathId(nextPathId++); + objDesc.SetPath(objPath); + + auto& objSelf = *objDesc.MutablePathDescription()->MutableSelf(); + objSelf.SetName(objName); objSelf.SetPathType(NKikimrSchemeOp::EPathTypeDir); - dirChildren.Add()->CopyFrom(objSelf); - - objSelf.SetPathId(objDesc.GetPathId()); - objSelf.SetSchemeshardId(objDesc.GetPathOwner()); - objSelf.SetCreateFinished(true); - objSelf.SetCreateTxId(1); - objSelf.SetCreateStep(1); - objSelf.SetParentPathId(dirDesc.GetPathId()); + dirChildren.Add()->CopyFrom(objSelf); + + objSelf.SetPathId(objDesc.GetPathId()); + objSelf.SetSchemeshardId(objDesc.GetPathOwner()); + objSelf.SetCreateFinished(true); + objSelf.SetCreateTxId(1); + objSelf.SetCreateStep(1); + objSelf.SetParentPathId(dirDesc.GetPathId()); objSelf.SetPathState(NKikimrSchemeOp::EPathStateNoChanges); - objSelf.SetPathVersion(version); - - descriptions[TPathId(owner, objDesc.GetPathId())] = std::move(objDescTwoPart); - } - - descriptions[TPathId(owner, dirDesc.GetPathId())] = std::move(dirDescTwoPart); - } - - return descriptions; - } - + objSelf.SetPathVersion(version); + + descriptions[TPathId(owner, objDesc.GetPathId())] = std::move(objDescTwoPart); + } + + descriptions[TPathId(owner, dirDesc.GetPathId())] = std::move(dirDescTwoPart); + } + + return descriptions; + } + TPathId RandomPathId(bool allowDir = true) const { - while (true) { + while (true) { const TPathId pathId(Owner, RandomNumber(NextPathId)); - - if (!Descriptions.contains(pathId)) { - continue; - } - - if (!allowDir && IsDir(Descriptions.at(pathId))) { - continue; - } - - return pathId; - } - - Y_FAIL("Unreachable"); - } - + + if (!Descriptions.contains(pathId)) { + continue; + } + + if (!allowDir && IsDir(Descriptions.at(pathId))) { + continue; + } + + return pathId; + } + + Y_FAIL("Unreachable"); + } + void Modify(TPathId pathId) { - Y_VERIFY(Descriptions.contains(pathId)); - - TDescription& description = Descriptions.at(pathId).Record; - auto& self = *description.MutablePathDescription()->MutableSelf(); - self.SetPathVersion(TlsActivationContext->Now().GetValue()); - - auto describeSchemeResult = MakeHolder<TDescribeSchemeResult>(); - describeSchemeResult->Record.CopyFrom(description); - Send(Populator, std::move(describeSchemeResult)); - - ++*ModifiedPaths; - } - + Y_VERIFY(Descriptions.contains(pathId)); + + TDescription& description = Descriptions.at(pathId).Record; + auto& self = *description.MutablePathDescription()->MutableSelf(); + self.SetPathVersion(TlsActivationContext->Now().GetValue()); + + auto describeSchemeResult = MakeHolder<TDescribeSchemeResult>(); + describeSchemeResult->Record.CopyFrom(description); + Send(Populator, std::move(describeSchemeResult)); + + ++*ModifiedPaths; + } + void Delete(TPathId pathId) { - Y_VERIFY(Descriptions.contains(pathId)); - - TDescription& description = Descriptions.at(pathId).Record; - Y_VERIFY(!IsDir(description)); - + Y_VERIFY(Descriptions.contains(pathId)); + + TDescription& description = Descriptions.at(pathId).Record; + Y_VERIFY(!IsDir(description)); + description.SetStatus(NKikimrScheme::StatusPathDoesNotExist); - - auto describeSchemeResult = MakeHolder<TDescribeSchemeResult>(); - describeSchemeResult->Record.CopyFrom(description); - Send(Populator, std::move(describeSchemeResult)); - + + auto describeSchemeResult = MakeHolder<TDescribeSchemeResult>(); + describeSchemeResult->Record.CopyFrom(description); + Send(Populator, std::move(describeSchemeResult)); + Modify(TPathId(description.GetPathDescription().GetSelf().GetSchemeshardId(), description.GetPathDescription().GetSelf().GetParentPathId())); - Descriptions.erase(pathId); - - ++*DeletedPaths; - } - - void Modify() { - Modify(RandomPathId()); - - const TDuration delta = TDuration::MicroSeconds(1000000 / Config.InFlightModifications); - Schedule(delta, new TEvents::TEvWakeup(TAG_MODIFY)); - } - - void Change() { - const bool create = RandomNumber((ui8)2); - - if (create) { - //Create(++NextPathId); - } else { - Delete(RandomPathId(false)); - } - - const TDuration delta = TDuration::MicroSeconds(1000000 / Config.InFlightChanges); - Schedule(delta, new TEvents::TEvWakeup(TAG_CHANGE)); - } - - void Boot() { + Descriptions.erase(pathId); + + ++*DeletedPaths; + } + + void Modify() { + Modify(RandomPathId()); + + const TDuration delta = TDuration::MicroSeconds(1000000 / Config.InFlightModifications); + Schedule(delta, new TEvents::TEvWakeup(TAG_MODIFY)); + } + + void Change() { + const bool create = RandomNumber((ui8)2); + + if (create) { + //Create(++NextPathId); + } else { + Delete(RandomPathId(false)); + } + + const TDuration delta = TDuration::MicroSeconds(1000000 / Config.InFlightChanges); + Schedule(delta, new TEvents::TEvWakeup(TAG_CHANGE)); + } + + void Boot() { const TActorId proxy = MakeStateStorageProxyID(StateStorageGroupFromTabletID(Owner)); - Send(proxy, new TEvStateStorage::TEvListSchemeBoard(), IEventHandle::FlagTrackDelivery); - - Become(&TThis::StateBoot); - } - - void Populate() { + Send(proxy, new TEvStateStorage::TEvListSchemeBoard(), IEventHandle::FlagTrackDelivery); + + Become(&TThis::StateBoot); + } + + void Populate() { const ui32 ssId = StateStorageGroupFromTabletID(Owner); - Descriptions = GenerateDescriptions(Owner, Config, NextPathId); - Populator = Register(CreateSchemeBoardPopulator( + Descriptions = GenerateDescriptions(Owner, Config, NextPathId); + Populator = Register(CreateSchemeBoardPopulator( Owner, Max<ui64>(), ssId, Descriptions, NextPathId - )); - + )); + TPathId pathId(Owner, NextPathId - 1); Y_VERIFY(Descriptions.contains(pathId)); - const TString& topPath = Descriptions.at(pathId).Record.GetPath(); + const TString& topPath = Descriptions.at(pathId).Record.GetPath(); - // subscriber will help us to know when sync is completed - Subscriber = Register(CreateSchemeBoardSubscriber( + // subscriber will help us to know when sync is completed + Subscriber = Register(CreateSchemeBoardSubscriber( SelfId(), topPath, ssId, - ESchemeBoardSubscriberDeletionPolicy::Majority - )); - + ESchemeBoardSubscriberDeletionPolicy::Majority + )); + *TotalPaths = NextPathId - 1 - NSchemeShard::RootPathId; - - Become(&TThis::StatePopulate); - } - - void Test() { - Modify(); - //Change(); - - Become(&TThis::StateTest); - } - - void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { - if (!ev->Get()->Info) { - return Boot(); - } - - Populate(); - } - - void Handle(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { - const auto* msg = ev->Get(); - - Y_VERIFY(ev->Sender == Subscriber); - Y_VERIFY(msg->PathId.LocalPathId == NextPathId - 1); - - Send(Subscriber, new TEvents::TEvPoisonPill()); + + Become(&TThis::StatePopulate); + } + + void Test() { + Modify(); + //Change(); + + Become(&TThis::StateTest); + } + + void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { + if (!ev->Get()->Info) { + return Boot(); + } + + Populate(); + } + + void Handle(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { + const auto* msg = ev->Get(); + + Y_VERIFY(ev->Sender == Subscriber); + Y_VERIFY(msg->PathId.LocalPathId == NextPathId - 1); + + Send(Subscriber, new TEvents::TEvPoisonPill()); Subscriber = TActorId(); - - const TInstant ts = TInstant::FromValue(GetPathVersion(msg->DescribeSchemeResult)); - *SyncDuration = (TlsActivationContext->Now() - ts).MilliSeconds(); - - Test(); - } - - void Handle(TEvents::TEvWakeup::TPtr& ev) { - switch (ev->Get()->Tag) { - case TAG_MODIFY: - Modify(); - break; - - case TAG_CHANGE: - Change(); - break; - - default: - Y_VERIFY_DEBUG(false, "Unknown wakeup tag"); - break; - } - } - - void PassAway() override { - Send(Populator, new TEvents::TEvPoisonPill()); - if (Subscriber) { - Send(Subscriber, new TEvents::TEvPoisonPill()); - } - - TActor::PassAway(); - } - -public: - explicit TLoadProducer(ui64 owner, const TTestConfig& config) - : Owner(owner) - , Config(config) + + const TInstant ts = TInstant::FromValue(GetPathVersion(msg->DescribeSchemeResult)); + *SyncDuration = (TlsActivationContext->Now() - ts).MilliSeconds(); + + Test(); + } + + void Handle(TEvents::TEvWakeup::TPtr& ev) { + switch (ev->Get()->Tag) { + case TAG_MODIFY: + Modify(); + break; + + case TAG_CHANGE: + Change(); + break; + + default: + Y_VERIFY_DEBUG(false, "Unknown wakeup tag"); + break; + } + } + + void PassAway() override { + Send(Populator, new TEvents::TEvPoisonPill()); + if (Subscriber) { + Send(Subscriber, new TEvents::TEvPoisonPill()); + } + + TActor::PassAway(); + } + +public: + explicit TLoadProducer(ui64 owner, const TTestConfig& config) + : Owner(owner) + , Config(config) , NextPathId(NSchemeShard::RootPathId + 1) - { - SyncDuration = Config.Counters->GetCounter("Producer/SyncDuration", false); - TotalPaths = Config.Counters->GetCounter("Producer/TotalPaths", false); - ModifiedPaths = Config.Counters->GetCounter("Producer/ModifiedPaths", false); - CreatedPaths = Config.Counters->GetCounter("Producer/CreatedPaths", false); - DeletedPaths = Config.Counters->GetCounter("Producer/DeletedPaths", false); - } - - void Bootstrap() { - Boot(); - } - - STATEFN(StateBoot) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvStateStorage::TEvListSchemeBoardResult, Handle); - cFunc(TEvents::TEvUndelivered::EventType, Boot); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StatePopulate) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvNotifyUpdate, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateTest) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvents::TEvWakeup, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - const ui64 Owner; - const TTestConfig Config; - NMonitoring::TDynamicCounters::TCounterPtr SyncDuration; - NMonitoring::TDynamicCounters::TCounterPtr TotalPaths; - NMonitoring::TDynamicCounters::TCounterPtr ModifiedPaths; - NMonitoring::TDynamicCounters::TCounterPtr CreatedPaths; - NMonitoring::TDynamicCounters::TCounterPtr DeletedPaths; - - TDescriptions Descriptions; - ui64 NextPathId; + { + SyncDuration = Config.Counters->GetCounter("Producer/SyncDuration", false); + TotalPaths = Config.Counters->GetCounter("Producer/TotalPaths", false); + ModifiedPaths = Config.Counters->GetCounter("Producer/ModifiedPaths", false); + CreatedPaths = Config.Counters->GetCounter("Producer/CreatedPaths", false); + DeletedPaths = Config.Counters->GetCounter("Producer/DeletedPaths", false); + } + + void Bootstrap() { + Boot(); + } + + STATEFN(StateBoot) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvStateStorage::TEvListSchemeBoardResult, Handle); + cFunc(TEvents::TEvUndelivered::EventType, Boot); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StatePopulate) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvNotifyUpdate, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateTest) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvWakeup, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const ui64 Owner; + const TTestConfig Config; + NMonitoring::TDynamicCounters::TCounterPtr SyncDuration; + NMonitoring::TDynamicCounters::TCounterPtr TotalPaths; + NMonitoring::TDynamicCounters::TCounterPtr ModifiedPaths; + NMonitoring::TDynamicCounters::TCounterPtr CreatedPaths; + NMonitoring::TDynamicCounters::TCounterPtr DeletedPaths; + + TDescriptions Descriptions; + ui64 NextPathId; TActorId Populator; TActorId Subscriber; - -}; // TLoadProducer - -class TLoadConsumer: public TActorBootstrapped<TLoadConsumer> { - void Subscribe(const TPathId& pathId) { + +}; // TLoadProducer + +class TLoadConsumer: public TActorBootstrapped<TLoadConsumer> { + void Subscribe(const TPathId& pathId) { const ui32 ssId = StateStorageGroupFromTabletID(Owner); - for (ui32 i = 0; i < Config.SubscriberMulti; ++i) { + for (ui32 i = 0; i < Config.SubscriberMulti; ++i) { const TActorId subscriber = Register(CreateSchemeBoardSubscriber( SelfId(), pathId, ssId, - ESchemeBoardSubscriberDeletionPolicy::Majority - )); - - Subscribers.push_back(subscriber); - ++*SubscribersCount; - } - } - - void BatchSubscribe(ui64 nextPathId, ui32 batchSizeLimit = 100) { - ui32 batchSize = 0; - while (nextPathId <= MaxPathId && batchSize++ < batchSizeLimit) { - Subscribe(TPathId(Owner, nextPathId++)); - } - - if (nextPathId > MaxPathId) { - return; - } - - Schedule(TDuration::MilliSeconds(20), new TEvents::TEvWakeup(nextPathId)); - } - - void Handle(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { - const auto* msg = ev->Get(); - - const TInstant ts = TInstant::FromValue(GetPathVersion(msg->DescribeSchemeResult)); - if (IsDir(msg->DescribeSchemeResult)) { - LatencyDir->Collect((TlsActivationContext->Now() - ts).MilliSeconds()); - } else { - Latency->Collect((TlsActivationContext->Now() - ts).MilliSeconds()); - } - } - - void Handle(TEvents::TEvWakeup::TPtr& ev) { - BatchSubscribe(ev->Get()->Tag); - } - - void PassAway() override { - for (const auto& subscriber : Subscribers) { - Send(subscriber, new TEvents::TEvPoisonPill()); - } - - TActor::PassAway(); - } - -public: - explicit TLoadConsumer(ui64 owner, const TTestConfig& config) - : Owner(owner) - , Config(config) - { + ESchemeBoardSubscriberDeletionPolicy::Majority + )); + + Subscribers.push_back(subscriber); + ++*SubscribersCount; + } + } + + void BatchSubscribe(ui64 nextPathId, ui32 batchSizeLimit = 100) { + ui32 batchSize = 0; + while (nextPathId <= MaxPathId && batchSize++ < batchSizeLimit) { + Subscribe(TPathId(Owner, nextPathId++)); + } + + if (nextPathId > MaxPathId) { + return; + } + + Schedule(TDuration::MilliSeconds(20), new TEvents::TEvWakeup(nextPathId)); + } + + void Handle(TSchemeBoardEvents::TEvNotifyUpdate::TPtr& ev) { + const auto* msg = ev->Get(); + + const TInstant ts = TInstant::FromValue(GetPathVersion(msg->DescribeSchemeResult)); + if (IsDir(msg->DescribeSchemeResult)) { + LatencyDir->Collect((TlsActivationContext->Now() - ts).MilliSeconds()); + } else { + Latency->Collect((TlsActivationContext->Now() - ts).MilliSeconds()); + } + } + + void Handle(TEvents::TEvWakeup::TPtr& ev) { + BatchSubscribe(ev->Get()->Tag); + } + + void PassAway() override { + for (const auto& subscriber : Subscribers) { + Send(subscriber, new TEvents::TEvPoisonPill()); + } + + TActor::PassAway(); + } + +public: + explicit TLoadConsumer(ui64 owner, const TTestConfig& config) + : Owner(owner) + , Config(config) + { MaxPathId = NSchemeShard::RootPathId - + Config.Dirs - + Config.Dirs * Config.ObjectsPerDir; - - Latency = Config.Counters->GetHistogram("Consumer/Latency", NMonitoring::ExponentialHistogram(15, 2, 1)); - LatencyDir = Config.Counters->GetHistogram("Consumer/LatencyDir", NMonitoring::ExponentialHistogram(10, 4, 1)); - SubscribersCount = Config.Counters->GetCounter("Consumer/Subscribers", false); - } - - void Bootstrap() { + + Config.Dirs + + Config.Dirs * Config.ObjectsPerDir; + + Latency = Config.Counters->GetHistogram("Consumer/Latency", NMonitoring::ExponentialHistogram(15, 2, 1)); + LatencyDir = Config.Counters->GetHistogram("Consumer/LatencyDir", NMonitoring::ExponentialHistogram(10, 4, 1)); + SubscribersCount = Config.Counters->GetCounter("Consumer/Subscribers", false); + } + + void Bootstrap() { BatchSubscribe(NSchemeShard::RootPathId + 1); - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvNotifyUpdate, Handle); - hFunc(TEvents::TEvWakeup, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - const ui64 Owner; - const TTestConfig Config; - NMonitoring::THistogramPtr Latency; - NMonitoring::THistogramPtr LatencyDir; - NMonitoring::TDynamicCounters::TCounterPtr SubscribersCount; - - ui64 MaxPathId; + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvNotifyUpdate, Handle); + hFunc(TEvents::TEvWakeup, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const ui64 Owner; + const TTestConfig Config; + NMonitoring::THistogramPtr Latency; + NMonitoring::THistogramPtr LatencyDir; + NMonitoring::TDynamicCounters::TCounterPtr SubscribersCount; + + ui64 MaxPathId; TVector<TActorId> Subscribers; - -}; // TLoadConsumer - -} // NSchemeBoard - -IActor* CreateSchemeBoardLoadProducer(ui64 owner, const NSchemeBoard::TTestConfig& config) { - return new NSchemeBoard::TLoadProducer(owner, config); -} - -IActor* CreateSchemeBoardLoadConsumer(ui64 owner, const NSchemeBoard::TTestConfig& config) { - return new NSchemeBoard::TLoadConsumer(owner, config); -} - -} // NKikimr + +}; // TLoadConsumer + +} // NSchemeBoard + +IActor* CreateSchemeBoardLoadProducer(ui64 owner, const NSchemeBoard::TTestConfig& config) { + return new NSchemeBoard::TLoadProducer(owner, config); +} + +IActor* CreateSchemeBoardLoadConsumer(ui64 owner, const NSchemeBoard::TTestConfig& config) { + return new NSchemeBoard::TLoadConsumer(owner, config); +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/load_test.h b/ydb/core/tx/scheme_board/load_test.h index f53b3496146..271e96bf3fc 100644 --- a/ydb/core/tx/scheme_board/load_test.h +++ b/ydb/core/tx/scheme_board/load_test.h @@ -1,38 +1,38 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <library/cpp/monlib/dynamic_counters/counters.h> - -namespace NKikimr { -namespace NSchemeBoard { - -struct TTestConfig { - NMonitoring::TDynamicCounterPtr Counters; - - ui64 Dirs; - ui64 ObjectsPerDir; - ui32 SubscriberMulti; - // modifications of existent objects - ui32 InFlightModifications; - // creations/deletions - ui32 InFlightChanges; - - explicit TTestConfig(NMonitoring::TDynamicCounterPtr counters) - : Counters(counters) - , Dirs(10) - , ObjectsPerDir(10000) - , SubscriberMulti(1) - , InFlightModifications(100) - , InFlightChanges(100) - { - } - -}; // TTestConfig - -} // NSchemeBoard - -IActor* CreateSchemeBoardLoadProducer(ui64 owner, const NSchemeBoard::TTestConfig& config); -IActor* CreateSchemeBoardLoadConsumer(ui64 owner, const NSchemeBoard::TTestConfig& config); - -} // NKikimr + +namespace NKikimr { +namespace NSchemeBoard { + +struct TTestConfig { + NMonitoring::TDynamicCounterPtr Counters; + + ui64 Dirs; + ui64 ObjectsPerDir; + ui32 SubscriberMulti; + // modifications of existent objects + ui32 InFlightModifications; + // creations/deletions + ui32 InFlightChanges; + + explicit TTestConfig(NMonitoring::TDynamicCounterPtr counters) + : Counters(counters) + , Dirs(10) + , ObjectsPerDir(10000) + , SubscriberMulti(1) + , InFlightModifications(100) + , InFlightChanges(100) + { + } + +}; // TTestConfig + +} // NSchemeBoard + +IActor* CreateSchemeBoardLoadProducer(ui64 owner, const NSchemeBoard::TTestConfig& config); +IActor* CreateSchemeBoardLoadConsumer(ui64 owner, const NSchemeBoard::TTestConfig& config); + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/mon_events.h b/ydb/core/tx/scheme_board/mon_events.h index fd63992f291..1e449dc2687 100644 --- a/ydb/core/tx/scheme_board/mon_events.h +++ b/ydb/core/tx/scheme_board/mon_events.h @@ -1,94 +1,94 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/base/pathid.h> #include <ydb/core/protos/scheme_board_mon.pb.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/json/json_value.h> - -namespace NKikimr { - -struct TSchemeBoardMonEvents { - enum EEv { - EvRegister = EventSpaceBegin(TKikimrEvents::ES_SCHEME_BOARD_MON), - EvUnregister, - - EvInfoRequest, - EvInfoResponse, - - EvDescribeRequest, - EvDescribeResponse, - - EvEnd, - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_BOARD_MON), "expect End < EventSpaceEnd(ES_SCHEME_BOARD_MON)"); - - struct TEvRegister: public TEventLocal<TEvRegister, EvRegister> { - const NKikimrServices::TActivity::EType ActivityType; - const NJson::TJsonMap Attributes; - - explicit TEvRegister( - NKikimrServices::TActivity::EType activityType, - const NJson::TJsonMap& attributes) - : ActivityType(activityType) - , Attributes(attributes) - { - } - }; - - struct TEvUnregister: public TEventLocal<TEvUnregister, EvUnregister> { - }; - - struct TEvInfoRequest: public TEventPB<TEvInfoRequest, NKikimrSchemeBoardMon::TEvInfoRequest, EvInfoRequest> { - TEvInfoRequest() = default; - - explicit TEvInfoRequest(ui32 limit) { - Record.SetLimitRepeatedFields(limit); - } - }; - - struct TEvInfoResponse: public TEventPB<TEvInfoResponse, NKikimrSchemeBoardMon::TEvInfoResponse, EvInfoResponse> { - TEvInfoResponse() = default; - - explicit TEvInfoResponse(const TActorId& self, const TString& activityType) { - ActorIdToProto(self, Record.MutableSelf()); - Record.SetActivityType(activityType); - } - - explicit TEvInfoResponse(const TActorId& self, NKikimrServices::TActivity::EType activityType) - : TEvInfoResponse(self, NKikimrServices::TActivity::EType_Name(activityType)) - { - } - - void SetTruncated(bool value = true) { - Record.SetTruncated(value); - } - }; - - struct TEvDescribeRequest: public TEventPB<TEvDescribeRequest, NKikimrSchemeBoardMon::TEvDescribeRequest, EvDescribeRequest> { - TEvDescribeRequest() = default; - - explicit TEvDescribeRequest(const TString& path) { - Record.SetPath(path); - } - - explicit TEvDescribeRequest(const TPathId& pathId) { - Record.MutablePathId()->SetOwnerId(pathId.OwnerId); - Record.MutablePathId()->SetLocalPathId(pathId.LocalPathId); - } - }; - - struct TEvDescribeResponse: public TEventPB<TEvDescribeResponse, NKikimrSchemeBoardMon::TEvDescribeResponse, EvDescribeResponse> { - TEvDescribeResponse() = default; - - explicit TEvDescribeResponse(const TString& json) { - Record.SetJson(json); - } - }; - -}; // TSchemeBoardMonEvents - -} // NKikimr + +#include <library/cpp/json/json_value.h> + +namespace NKikimr { + +struct TSchemeBoardMonEvents { + enum EEv { + EvRegister = EventSpaceBegin(TKikimrEvents::ES_SCHEME_BOARD_MON), + EvUnregister, + + EvInfoRequest, + EvInfoResponse, + + EvDescribeRequest, + EvDescribeResponse, + + EvEnd, + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_BOARD_MON), "expect End < EventSpaceEnd(ES_SCHEME_BOARD_MON)"); + + struct TEvRegister: public TEventLocal<TEvRegister, EvRegister> { + const NKikimrServices::TActivity::EType ActivityType; + const NJson::TJsonMap Attributes; + + explicit TEvRegister( + NKikimrServices::TActivity::EType activityType, + const NJson::TJsonMap& attributes) + : ActivityType(activityType) + , Attributes(attributes) + { + } + }; + + struct TEvUnregister: public TEventLocal<TEvUnregister, EvUnregister> { + }; + + struct TEvInfoRequest: public TEventPB<TEvInfoRequest, NKikimrSchemeBoardMon::TEvInfoRequest, EvInfoRequest> { + TEvInfoRequest() = default; + + explicit TEvInfoRequest(ui32 limit) { + Record.SetLimitRepeatedFields(limit); + } + }; + + struct TEvInfoResponse: public TEventPB<TEvInfoResponse, NKikimrSchemeBoardMon::TEvInfoResponse, EvInfoResponse> { + TEvInfoResponse() = default; + + explicit TEvInfoResponse(const TActorId& self, const TString& activityType) { + ActorIdToProto(self, Record.MutableSelf()); + Record.SetActivityType(activityType); + } + + explicit TEvInfoResponse(const TActorId& self, NKikimrServices::TActivity::EType activityType) + : TEvInfoResponse(self, NKikimrServices::TActivity::EType_Name(activityType)) + { + } + + void SetTruncated(bool value = true) { + Record.SetTruncated(value); + } + }; + + struct TEvDescribeRequest: public TEventPB<TEvDescribeRequest, NKikimrSchemeBoardMon::TEvDescribeRequest, EvDescribeRequest> { + TEvDescribeRequest() = default; + + explicit TEvDescribeRequest(const TString& path) { + Record.SetPath(path); + } + + explicit TEvDescribeRequest(const TPathId& pathId) { + Record.MutablePathId()->SetOwnerId(pathId.OwnerId); + Record.MutablePathId()->SetLocalPathId(pathId.LocalPathId); + } + }; + + struct TEvDescribeResponse: public TEventPB<TEvDescribeResponse, NKikimrSchemeBoardMon::TEvDescribeResponse, EvDescribeResponse> { + TEvDescribeResponse() = default; + + explicit TEvDescribeResponse(const TString& json) { + Record.SetJson(json); + } + }; + +}; // TSchemeBoardMonEvents + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/monitorable_actor.h b/ydb/core/tx/scheme_board/monitorable_actor.h index 953bc98b928..1db3eae7ccf 100644 --- a/ydb/core/tx/scheme_board/monitorable_actor.h +++ b/ydb/core/tx/scheme_board/monitorable_actor.h @@ -1,51 +1,51 @@ -#pragma once - -#include "mon_events.h" -#include "monitoring.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> - -#include <library/cpp/json/json_value.h> - -#include <util/string/printf.h> - -namespace NKikimr { -namespace NSchemeBoard { - -template <typename TDerived> -class TMonitorableActor: public TActorBootstrapped<TDerived> { - void MonRegister() { - this->Send(MakeSchemeBoardMonitoringId(), new TSchemeBoardMonEvents::TEvRegister( - TDerived::ActorActivityType(), this->MonAttributes())); - } - - void MonUnregister() { - this->Send(MakeSchemeBoardMonitoringId(), new TSchemeBoardMonEvents::TEvUnregister()); - } - -protected: - virtual NJson::TJsonMap MonAttributes() const { - return {}; - } - - static NJson::TJsonMap PrintActorIdAttr(NKikimrServices::TActivity::EType activityType, const TActorId& actorId) { - return NJson::TJsonMap({ - {"@type", "ACTOR_ID"}, - {"ActivityType", NKikimrServices::TActivity::EType_Name(activityType)}, - {"ActorId", Sprintf("%" PRIu64 ":%" PRIu64, actorId.RawX1(), actorId.RawX2())}, - }); - } - -public: - void Bootstrap() { - MonRegister(); - } - - void PassAway() override { - MonUnregister(); - IActor::PassAway(); - } -}; - -} // NSchemeBoard -} // NKikimr +#pragma once + +#include "mon_events.h" +#include "monitoring.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> + +#include <library/cpp/json/json_value.h> + +#include <util/string/printf.h> + +namespace NKikimr { +namespace NSchemeBoard { + +template <typename TDerived> +class TMonitorableActor: public TActorBootstrapped<TDerived> { + void MonRegister() { + this->Send(MakeSchemeBoardMonitoringId(), new TSchemeBoardMonEvents::TEvRegister( + TDerived::ActorActivityType(), this->MonAttributes())); + } + + void MonUnregister() { + this->Send(MakeSchemeBoardMonitoringId(), new TSchemeBoardMonEvents::TEvUnregister()); + } + +protected: + virtual NJson::TJsonMap MonAttributes() const { + return {}; + } + + static NJson::TJsonMap PrintActorIdAttr(NKikimrServices::TActivity::EType activityType, const TActorId& actorId) { + return NJson::TJsonMap({ + {"@type", "ACTOR_ID"}, + {"ActivityType", NKikimrServices::TActivity::EType_Name(activityType)}, + {"ActorId", Sprintf("%" PRIu64 ":%" PRIu64, actorId.RawX1(), actorId.RawX2())}, + }); + } + +public: + void Bootstrap() { + MonRegister(); + } + + void PassAway() override { + MonUnregister(); + IActor::PassAway(); + } +}; + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/monitoring.cpp b/ydb/core/tx/scheme_board/monitoring.cpp index 956e31c12f8..deda6046552 100644 --- a/ydb/core/tx/scheme_board/monitoring.cpp +++ b/ydb/core/tx/scheme_board/monitoring.cpp @@ -1,1352 +1,1352 @@ -#include "mon_events.h" -#include "monitoring.h" - +#include "mon_events.h" +#include "monitoring.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/pathid.h> #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/mon/mon.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/mon.h> - -#include <library/cpp/monlib/service/pages/mon_page.h> -#include <library/cpp/monlib/service/pages/templates.h> - -#include <library/cpp/json/json_value.h> -#include <library/cpp/json/json_writer.h> - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/list.h> -#include <util/generic/variant.h> -#include <util/stream/str.h> -#include <util/string/builder.h> -#include <util/string/cast.h> -#include <util/string/join.h> -#include <util/string/split.h> - -// additional html elements -namespace NMonitoring { - const char NavTag[] = "nav"; - using TNav = TTag<NavTag>; -} - -namespace NKikimr { -namespace NSchemeBoard { - -using namespace NJson; - -class TMonitoring: public TActorBootstrapped<TMonitoring> { - static constexpr char ROOT[] = "scheme_board"; - - using TActivity = NKikimrServices::TActivity; - using EActivityType = TActivity::EType; - using EContentType = NMon::IEvHttpInfoRes::EContentType; - - enum class ERequestType { - Unknown, - Index, - Populator, - ReplicaPopulator, - Replica, - Subscriber, - SubscriberProxy, - ReplicaSubscriber, - Cache, - Describe, - Resolver, - Resolve, - }; - - enum class EAttributeType { - Unknown, - Pod, - String, - ActorId, - }; - - struct TActorInfo { - EActivityType ActivityType; - TJsonMap Attributes; - - TActorInfo(EActivityType activityType, const TJsonMap& attributes) - : ActivityType(activityType) - , Attributes(attributes) - { - } - - TActorInfo(const TSchemeBoardMonEvents::TEvRegister& ev) - : TActorInfo(ev.ActivityType, ev.Attributes) - { - } - }; - - void Handle(TSchemeBoardMonEvents::TEvRegister::TPtr& ev) { - const auto& msg = *ev->Get(); - - if (RegisteredActors.emplace(ev->Sender, msg).second) { - ByActivityType[msg.ActivityType].emplace(ev->Sender); - } - } - - void Handle(TSchemeBoardMonEvents::TEvUnregister::TPtr& ev) { - auto registered = RegisteredActors.find(ev->Sender); - if (registered == RegisteredActors.end()) { - return; - } - - auto byActivity = ByActivityType.find(registered->second.ActivityType); - Y_VERIFY(byActivity != ByActivityType.end()); - - byActivity->second.erase(ev->Sender); - if (byActivity->second.empty()) { - ByActivityType.erase(byActivity); - } - - RegisteredActors.erase(registered); - } - - static ERequestType ParseRequestType(const TStringBuf relPath) { - if (!relPath || relPath == "/") { - return ERequestType::Index; - } else if (relPath.StartsWith("/populator")) { - return ERequestType::Populator; - } else if (relPath.StartsWith("/replica_populator")) { - return ERequestType::ReplicaPopulator; - } else if (relPath.StartsWith("/replica")) { - return ERequestType::Replica; - } else if (relPath.StartsWith("/subscriber")) { - return ERequestType::Subscriber; - } else if (relPath.StartsWith("/subscriber_proxy")) { - return ERequestType::SubscriberProxy; - } else if (relPath.StartsWith("/replica_subscriber")) { - return ERequestType::ReplicaSubscriber; - } else if (relPath.StartsWith("/cache")) { - return ERequestType::Cache; - } else if (relPath.StartsWith("/describe")) { - return ERequestType::Describe; - } else if (relPath.StartsWith("/resolver")) { - return ERequestType::Resolver; - } else if (relPath.StartsWith("/resolve")) { - return ERequestType::Resolve; - } else { - return ERequestType::Unknown; - } - } - - static TString MakeLink(ERequestType requestType, const TStringBuf prefix) { - TStringBuilder str; - - if (prefix) { - str << prefix << "/"; - } - - switch (requestType) { - case ERequestType::Index: - return str << ROOT; - case ERequestType::Populator: - return str << "populator"; - case ERequestType::ReplicaPopulator: - return str << "replica_populator"; - case ERequestType::Replica: - return str << "replica"; - case ERequestType::Subscriber: - return str << "subscriber"; - case ERequestType::SubscriberProxy: - return str << "subscriber_proxy"; - case ERequestType::ReplicaSubscriber: - return str << "replica_subscriber"; - case ERequestType::Cache: - return str << "cache"; - case ERequestType::Describe: - return str << "describe"; - case ERequestType::Resolver: - return str << "resolver"; - case ERequestType::Resolve: - return str << "resolve"; - case ERequestType::Unknown: - return str; - } - } - - static EAttributeType ParseAttributeType(const TJsonValue& value) { - switch (value.GetType()) { - case JSON_BOOLEAN: - case JSON_INTEGER: - case JSON_DOUBLE: - case JSON_UINTEGER: - return EAttributeType::Pod; - - case JSON_STRING: - return EAttributeType::String; - + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/mon.h> + +#include <library/cpp/monlib/service/pages/mon_page.h> +#include <library/cpp/monlib/service/pages/templates.h> + +#include <library/cpp/json/json_value.h> +#include <library/cpp/json/json_writer.h> + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/list.h> +#include <util/generic/variant.h> +#include <util/stream/str.h> +#include <util/string/builder.h> +#include <util/string/cast.h> +#include <util/string/join.h> +#include <util/string/split.h> + +// additional html elements +namespace NMonitoring { + const char NavTag[] = "nav"; + using TNav = TTag<NavTag>; +} + +namespace NKikimr { +namespace NSchemeBoard { + +using namespace NJson; + +class TMonitoring: public TActorBootstrapped<TMonitoring> { + static constexpr char ROOT[] = "scheme_board"; + + using TActivity = NKikimrServices::TActivity; + using EActivityType = TActivity::EType; + using EContentType = NMon::IEvHttpInfoRes::EContentType; + + enum class ERequestType { + Unknown, + Index, + Populator, + ReplicaPopulator, + Replica, + Subscriber, + SubscriberProxy, + ReplicaSubscriber, + Cache, + Describe, + Resolver, + Resolve, + }; + + enum class EAttributeType { + Unknown, + Pod, + String, + ActorId, + }; + + struct TActorInfo { + EActivityType ActivityType; + TJsonMap Attributes; + + TActorInfo(EActivityType activityType, const TJsonMap& attributes) + : ActivityType(activityType) + , Attributes(attributes) + { + } + + TActorInfo(const TSchemeBoardMonEvents::TEvRegister& ev) + : TActorInfo(ev.ActivityType, ev.Attributes) + { + } + }; + + void Handle(TSchemeBoardMonEvents::TEvRegister::TPtr& ev) { + const auto& msg = *ev->Get(); + + if (RegisteredActors.emplace(ev->Sender, msg).second) { + ByActivityType[msg.ActivityType].emplace(ev->Sender); + } + } + + void Handle(TSchemeBoardMonEvents::TEvUnregister::TPtr& ev) { + auto registered = RegisteredActors.find(ev->Sender); + if (registered == RegisteredActors.end()) { + return; + } + + auto byActivity = ByActivityType.find(registered->second.ActivityType); + Y_VERIFY(byActivity != ByActivityType.end()); + + byActivity->second.erase(ev->Sender); + if (byActivity->second.empty()) { + ByActivityType.erase(byActivity); + } + + RegisteredActors.erase(registered); + } + + static ERequestType ParseRequestType(const TStringBuf relPath) { + if (!relPath || relPath == "/") { + return ERequestType::Index; + } else if (relPath.StartsWith("/populator")) { + return ERequestType::Populator; + } else if (relPath.StartsWith("/replica_populator")) { + return ERequestType::ReplicaPopulator; + } else if (relPath.StartsWith("/replica")) { + return ERequestType::Replica; + } else if (relPath.StartsWith("/subscriber")) { + return ERequestType::Subscriber; + } else if (relPath.StartsWith("/subscriber_proxy")) { + return ERequestType::SubscriberProxy; + } else if (relPath.StartsWith("/replica_subscriber")) { + return ERequestType::ReplicaSubscriber; + } else if (relPath.StartsWith("/cache")) { + return ERequestType::Cache; + } else if (relPath.StartsWith("/describe")) { + return ERequestType::Describe; + } else if (relPath.StartsWith("/resolver")) { + return ERequestType::Resolver; + } else if (relPath.StartsWith("/resolve")) { + return ERequestType::Resolve; + } else { + return ERequestType::Unknown; + } + } + + static TString MakeLink(ERequestType requestType, const TStringBuf prefix) { + TStringBuilder str; + + if (prefix) { + str << prefix << "/"; + } + + switch (requestType) { + case ERequestType::Index: + return str << ROOT; + case ERequestType::Populator: + return str << "populator"; + case ERequestType::ReplicaPopulator: + return str << "replica_populator"; + case ERequestType::Replica: + return str << "replica"; + case ERequestType::Subscriber: + return str << "subscriber"; + case ERequestType::SubscriberProxy: + return str << "subscriber_proxy"; + case ERequestType::ReplicaSubscriber: + return str << "replica_subscriber"; + case ERequestType::Cache: + return str << "cache"; + case ERequestType::Describe: + return str << "describe"; + case ERequestType::Resolver: + return str << "resolver"; + case ERequestType::Resolve: + return str << "resolve"; + case ERequestType::Unknown: + return str; + } + } + + static EAttributeType ParseAttributeType(const TJsonValue& value) { + switch (value.GetType()) { + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_DOUBLE: + case JSON_UINTEGER: + return EAttributeType::Pod; + + case JSON_STRING: + return EAttributeType::String; + case JSON_MAP: { - const auto* type = value.GetMapSafe().FindPtr("@type"); - - if (!type || type->GetType() != JSON_STRING) { - return EAttributeType::Unknown; - } - - if (type->GetStringSafe() == "ACTOR_ID") { - return EAttributeType::ActorId; - } + const auto* type = value.GetMapSafe().FindPtr("@type"); + + if (!type || type->GetType() != JSON_STRING) { + return EAttributeType::Unknown; + } + + if (type->GetStringSafe() == "ACTOR_ID") { + return EAttributeType::ActorId; + } // can not detenmine map type, fallback to unknown [[fallthrough]]; - } - - default: - return EAttributeType::Unknown; - } - } - - template <typename P, typename D> - static bool TryGetNext(TStringBuf& s, D delim, P& param) { - TMaybe<TStringBuf> buf; - GetNext(s, delim, buf); - if (!buf) { - return false; - } - - return TryFromString(*buf, param); - } - - static TActorId ParseActorId(TStringBuf str) { - ui64 x1; - ui64 x2; - - if (!TryGetNext(str, ':', x1) || !TryGetNext(str, ':', x2)) { - return {}; - } - - return TActorId(x1, x2); - } - - static TActorId ParseActorIdFromPath(TStringBuf relPath) { - if (relPath.EndsWith('/')) { - relPath.Chop(1); - } - - auto lastPart = relPath.RNextTok('/'); - if (!lastPart) { - return {}; - } - - return ParseActorId(lastPart); - } - - static TActorId ParseActorId(const TJsonValue& value) { - if (value.GetType() != JSON_STRING) { - return {}; - } - - return ParseActorId(TStringBuf(value.GetStringSafe())); - } - - static EActivityType ParseActivityType(const TJsonValue& value) { - if (value.GetType() != JSON_STRING) { - return TActivity::OTHER; - } - - EActivityType result; - if (!TActivity::EType_Parse(value.GetStringSafe(), &result)) { - return TActivity::OTHER; - } - - return result; - } - - static std::pair<EActivityType, TActorId> ParseActorId(const TJsonValue::TMapType& map) { - const auto* activityType = map.FindPtr("ActivityType"); - const auto* actorId = map.FindPtr("ActorId"); - - if (!activityType || !actorId) { - return {}; - } - - return std::make_pair(ParseActivityType(*activityType), ParseActorId(*actorId)); - } - - static TPathId PathIdFromProto(const NKikimrSchemeBoardMon::TPathId& proto) { - return TPathId(proto.GetOwnerId(), proto.GetLocalPathId()); - } - - template <typename T> - static TString GetPath(const T& proto) { - if (proto.HasPath()) { - return proto.GetPath(); - } else if (proto.HasPathId()) { - return ToString(PathIdFromProto(proto.GetPathId())); - } - - return {}; - } - - const TJsonValue::TMapType& GetAttrs(const TActorId& actorId) const { - auto it = RegisteredActors.find(actorId); - Y_VERIFY(it != RegisteredActors.end()); - - return it->second.Attributes.GetMapSafe(); - } - - using TRenderer = std::function<void(IOutputStream&)>; - - template <typename T, typename U> - static void Header(IOutputStream& str, const T& title, const U& subTitile) { - HTML(str) { - DIV_CLASS("page-header") { - H3() { - str << title; - if (subTitile) { - SMALL() { str << " " << subTitile; } - } - } - } - } - } - - template <> - void Header(IOutputStream& str, const TString& activityType, const NActorsProto::TActorId& actorId) { - Header(str, activityType, ActorIdFromProto(actorId)); - } - - static void Alert(IOutputStream& str, const TStringBuf text) { - HTML(str) { - DIV_CLASS("alert alert-warning") { - STRONG() { - str << "Warning:"; - } - str << " " << text << "."; - } - } - } - - static void Panel(IOutputStream& str, TRenderer title, TRenderer body) { - HTML(str) { - DIV_CLASS("panel panel-default") { - DIV_CLASS("panel-heading") { - H4_CLASS("panel-title") { - title(str); - } - } - body(str); - } - } - } - - static void SimplePanel(IOutputStream& str, const TStringBuf title, TRenderer body) { - auto titleRenderer = [&title](IOutputStream& str) { - HTML(str) { - str << title; - } - }; - - auto bodyRenderer = [body = std::move(body)](IOutputStream& str) { - HTML(str) { - DIV_CLASS("panel-body") { - body(str); - } - } - }; - - Panel(str, titleRenderer, bodyRenderer); - } - - static void CollapsedPanel(IOutputStream& str, const TStringBuf title, const TStringBuf targetId, TRenderer body) { - auto titleRenderer = [&title, &targetId](IOutputStream& str) { - HTML(str) { - str << "<a data-toggle='collapse' href='#" << targetId << "'>" - << title - << "</a>"; - } - }; - - auto bodyRenderer = [&targetId, body = std::move(body)](IOutputStream& str) { - HTML(str) { - str << "<div id='" << targetId << "' class='collapse'>"; - DIV_CLASS("panel-body") { - body(str); - } - str << "</div>"; - } - }; - - Panel(str, titleRenderer, bodyRenderer); - } - - static ERequestType ConvertActivityType(EActivityType activityType) { - static THashMap<EActivityType, ERequestType> activityToRequest = { - {TActivity::SCHEME_BOARD_POPULATOR_ACTOR, ERequestType::Populator}, - {TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, ERequestType::ReplicaPopulator}, - {TActivity::SCHEME_BOARD_REPLICA_ACTOR, ERequestType::Replica}, - {TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, ERequestType::Subscriber}, - {TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, ERequestType::SubscriberProxy}, - {TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, ERequestType::ReplicaSubscriber}, - {TActivity::PROXY_SCHEME_CACHE, ERequestType::Cache}, - }; - - return activityToRequest.Value(activityType, ERequestType::Unknown); - } - - static TString ActorIdToStringSafe(const TActorId& actorId) { - return TStringBuilder() << actorId.RawX1() << ":" << actorId.RawX2(); - } - - template <typename T> - static void Link(IOutputStream& str, const TStringBuf path, const T& title) { - HTML(str) { - HREF(path) { - str << title; - } - } - } - - static void Link(IOutputStream& str, ERequestType requestType, const TStringBuf title, const TStringBuf prefix = "..") { - Link(str, MakeLink(requestType, prefix), title); - } - - static void Link(IOutputStream& str, EActivityType activityType, const TActorId& actorId, const TStringBuf prefix = "..") { - const TString path = TStringBuilder() - << MakeLink(ConvertActivityType(activityType), prefix) - << "/" << ActorIdToStringSafe(actorId); - Link(str, path, actorId); - } - - static void Link(IOutputStream& str, EActivityType activityType, const NActorsProto::TActorId& actorId, const TStringBuf prefix = "..") { - Link(str, activityType, ActorIdFromProto(actorId), prefix); - } - - template <typename T> - static void TermDesc(IOutputStream& str, const TStringBuf term, const T& desc) { - HTML(str) { - DT() { str << term; } - DD() { str << desc; } - } - } - - template <typename T> - static void TermDescLink(IOutputStream& str, const TStringBuf term, EActivityType activityType, const T& actorId) { - HTML(str) { - DT() { str << term; } - DD() { Link(str, activityType, actorId); } - } - } - - enum EFormType : ui8 { - ByPath = 1 << 0, - ByPathId = 1 << 1, - Both = ByPath | ByPathId, - }; - - static void Form(IOutputStream& str, EFormType formType, ERequestType linkType, const TStringBuf linkPrefix, const TActorId& actorId) { - HTML(str) { - FORM_CLASS("form-horizontal") { - const auto action = MakeLink(linkType, linkPrefix); - - if (formType & EFormType::ByPath) { - DIV_CLASS("form-group") { - LABEL_CLASS_FOR("col-sm-2 control-label", "path") { - str << "Path"; - } - DIV_CLASS("col-sm-8") { - str << "<input type='text' id='path' name='path' class='form-control' placeholder='/full/path'>"; - } - DIV_CLASS("col-sm-2") { - str << "<button type='submit' name='byPath' formaction='" << action << "' class='btn btn-primary'>" - << "Find by path" - << "</button>"; - } - } - } - - if (formType & EFormType::ByPathId) { - DIV_CLASS("form-group") { - LABEL_CLASS_FOR("col-sm-2 control-label", "pathId") { - str << "PathId"; - } - DIV_CLASS("col-sm-4") { - str << "<input type='number' id='ownerId' name='ownerId' class='form-control' placeholder='owner id'>"; - } - DIV_CLASS("col-sm-4") { - str << "<input type='number' id='pathId' name='pathId' class='form-control' placeholder='local path id'>"; - } - DIV_CLASS("col-sm-2") { - str << "<button type='submit' name='byPathId' formaction='" << action << "' class='btn btn-info'>" - << "Find by path id" - << "</button>"; - } - } - } - - if (actorId) { - str << "<input type='hidden' name='actorId' value='" << ActorIdToStringSafe(actorId) << "'>"; - } - - str << "<pre id='description' class='hidden'/>"; - - str << R"(<script> - $(document).ready(function() { - $('button').click(function(e) { - e.preventDefault(); - - var btn = this; - var form = $('form'); - - $.ajax({ - type: "GET", - url: btn.formAction, - data: Object.assign({[btn.name]: 1}, form.serializeArray().reduce(function(obj, cur, _) { - obj[cur.name] = cur.value; - return obj; - }, {})), - success: function(data) { - $('#description').text(JSON.stringify(data, null, ' ')).removeClass('hidden'); - }, - error: function (data) { - $('#description').text("Error: " + data).removeClass('hidden'); - }, - }); - }); - }); - </script>)"; - } - } - } - - static void ResolveForm(IOutputStream& str) { - Form(str, EFormType::Both, ERequestType::Resolve, "", {}); - } - - static void DescribeForm(IOutputStream& str, const TActorId& actorId, EFormType type = EFormType::Both) { - Form(str, type, ERequestType::Describe, "..", actorId); - } - - static void Navbar(IOutputStream& str, ERequestType originRequestType) { - static THashMap<ERequestType, TStringBuf> requestTypeToTitle = { - {ERequestType::Index, "Main"}, - {ERequestType::Resolver, "Resolver"}, - }; - - const bool isIndex = originRequestType == ERequestType::Index; - - HTML(str) { - TAG_CLASS(TNav, "navbar") { - UL_CLASS("nav nav-pills") { - for (const auto& [rt, title] : requestTypeToTitle) { - if (rt == originRequestType) { - LI_CLASS("active") { Link(str, "#", title); } - } else { - LI() { Link(str, rt, title, isIndex ? ROOT : ".."); } - } - } - } - } - } - } - - TString RenderIndex() const { - TStringStream str; - - HTML(str) { - Navbar(str, ERequestType::Index); - - for (const auto& kv : ByActivityType) { - const auto& activityType = kv.first; - const auto& actorIds = kv.second; - const auto activityTypeStr = TActivity::EType_Name(activityType); - - CollapsedPanel(str, activityTypeStr, activityTypeStr, [&](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - - Y_VERIFY(!actorIds.empty()); - for (const auto& [key, _] : GetAttrs(*actorIds.begin())) { - TABLEH() { str << key; } - } - } - } - TABLEBODY() { - int i = 1; - for (const auto& actorId : actorIds) { - TABLER() { - TABLED() { str << i++; } - TABLED() { Link(str, activityType, actorId, ROOT); } - - for (const auto& [_, value] : GetAttrs(actorId)) { - switch (ParseAttributeType(value)) { - case EAttributeType::Pod: - TABLED() { str << value; } - break; - - case EAttributeType::String: - TABLED() { str << value.GetStringSafe(); } - break; - - case EAttributeType::ActorId: { - const auto kv = ParseActorId(value.GetMapSafe()); - TABLED() { Link(str, kv.first, kv.second, ROOT); } - break; - } - - case EAttributeType::Unknown: - break; - } - } - } - } - } - } - } - }); - } - } - - return str.Str(); - } - - static TString RenderResolver(const TList<TActorId>& replicas) { - TStringStream str; - - HTML(str) { - Navbar(str, ERequestType::Resolver); - Header(str, "Replica resolver", ""); - - SimplePanel(str, "Resolver", [](IOutputStream& str) { - ResolveForm(str); - }); - - CollapsedPanel(str, "All replicas", "allReplicas", [&replicas](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - } - } - TABLEBODY() { - ui32 i = 0; - for (const auto& replica : replicas) { - TABLER() { - TABLED() { str << ++i; } - TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_ACTOR, replica, ""); } - } - } - } - } - } - }); - } - - return str.Str(); - } - - static TString RenderReplica(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaResponse); - const auto& response = record.GetReplicaResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - if (record.GetTruncated()) { - Alert(str, "some lists has been truncated"); - } - - SimplePanel(str, "Descriptions", [&record](IOutputStream& str) { - const auto& response = record.GetReplicaResponse(); - - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDesc(str, "TotalCount", response.GetDescriptions().GetTotalCount()); - TermDesc(str, "ByPathCount", response.GetDescriptions().GetByPathCount()); - TermDesc(str, "ByPathIdCount", response.GetDescriptions().GetByPathIdCount()); - } - - DescribeForm(str, ActorIdFromProto(record.GetSelf())); - } - }); - - CollapsedPanel(str, "Populators", "populators", [&response](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - TABLEH() { str << "Owner"; } - TABLEH() { str << "Generation"; } - TABLEH() { str << "PendingGeneration"; } - } - } - TABLEBODY() { - for (ui32 i = 0; i < response.PopulatorsSize(); ++i) { - const auto& populator = response.GetPopulators(i); - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, populator.GetActorId()); } - TABLED() { str << populator.GetOwner(); } - TABLED() { str << populator.GetGeneration(); } - TABLED() { str << populator.GetPendingGeneration(); } - } - } - } - } - } - }); - - CollapsedPanel(str, "Subscribers", "subscribers", [&response](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - TABLEH() { str << "Path"; } - } - } - TABLEBODY() { - for (ui32 i = 0; i < response.SubscribersSize(); ++i) { - const auto& subscriber = response.GetSubscribers(i); - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, subscriber.GetActorId()); } - TABLED() { str << GetPath(subscriber); } - } - } - } - } - } - }); - } - - return str.Str(); - } - - static TString RenderPopulator(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kPopulatorResponse); - const auto& response = record.GetPopulatorResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - if (record.GetTruncated()) { - Alert(str, "some lists has been truncated"); - } - - SimplePanel(str, "Info", [&record](IOutputStream& str) { - const auto& response = record.GetPopulatorResponse(); - - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDesc(str, "Owner", response.GetOwner()); - TermDesc(str, "Generation", response.GetGeneration()); - TermDesc(str, "DescriptionsCount", response.GetDescriptionsCount()); - TermDesc(str, "MaxPathId", PathIdFromProto(response.GetMaxPathId())); - TermDesc(str, "DelayedUpdatesCount", response.GetDelayedUpdatesCount()); - } - - DescribeForm(str, ActorIdFromProto(record.GetSelf()), EFormType::ByPathId); - } - }); - - CollapsedPanel(str, "ReplicaPopulators", "replicaPopulators", [&response](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - } - } - TABLEBODY() { - for (ui32 i = 0; i < response.ReplicaPopulatorsSize(); ++i) { - const auto& replicaPopulator = response.GetReplicaPopulators(i); - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, replicaPopulator); } - } - } - } - } - } - }); - - CollapsedPanel(str, "UpdateAcks", "updateAcks", [&response](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Cookie"; } - TABLEH() { str << "AckTo"; } - TABLEH() { str << "PathId / Version / AcksCount"; } - } - } - TABLEBODY() { - for (ui32 i = 0; i < response.UpdateAcksSize(); ++i) { - const auto& updateAck = response.GetUpdateAcks(i); - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { str << updateAck.GetCookie(); } - TABLED() { str << ActorIdFromProto(updateAck.GetAckTo()); } - TABLED() { - TABLE_CLASS("table table-condensed") { - TABLEBODY() { - for (ui32 i = 0; i < updateAck.PathAcksSize(); ++i) { - const auto& pathAck = updateAck.GetPathAcks(i); - - TABLER() { - TABLED() { str << PathIdFromProto(pathAck.GetPathId()); } - TABLED() { str << pathAck.GetVersion(); } - TABLED() { str << pathAck.GetAcksCount(); } - } - } - } - } - } - } - } - } - } - } - }); - } - - return str.Str(); - } - - static TString RenderReplicaPopulator(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaPopulatorResponse); - const auto& response = record.GetReplicaPopulatorResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - if (record.GetTruncated()) { - Alert(str, "some lists has been truncated"); - } - - SimplePanel(str, "Info", [&response](IOutputStream& str) { - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_POPULATOR_ACTOR, response.GetParent()); - TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); - TermDesc(str, "Owner", response.GetOwner()); - TermDesc(str, "Generation", response.GetGeneration()); - TermDesc(str, "CurPathId", PathIdFromProto(response.GetCurPathId())); - TermDesc(str, "LastAckedPathId", PathIdFromProto(response.GetLastAckedPathId())); - TermDesc(str, "BatchSize", response.GetBatchSize()); - TermDesc(str, "BatchSizeLimit", response.GetBatchSizeLimit()); - } - } - }); - - auto renderUpdates = [&str](const TStringBuf title, const TStringBuf targetId, const auto& updates) { - CollapsedPanel(str, title, targetId, [&updates](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "PathId"; } - TABLEH() { str << "Version / TxIds"; } - } - } - TABLEBODY() { - for (int i = 0; i < updates.size(); ++i) { - const auto& update = updates[i]; - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { str << PathIdFromProto(update.GetPathId()); } - TABLED() { - TABLE_CLASS("table table-condensed") { - TABLEBODY() { - for (ui32 i = 0; i < update.VersionsSize(); ++i) { - const auto& version = update.GetVersions(i); - - TABLER() { - TABLED() { str << version.GetVersion(); } - TABLED() { str << JoinSeq(", ", version.GetTxIds()); } - } - } - } - } - } - } - } - } - } - } - }); - }; - - renderUpdates("Updates", "updates", response.GetUpdates()); - renderUpdates("UpdatesRequested", "updatesRequested", response.GetUpdatesRequested()); - renderUpdates("UpdatesInFlight", "updatesInFlight", response.GetUpdatesInFlight()); - } - - return str.Str(); - } - - static TString RenderSubscriber(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberResponse); - const auto& response = record.GetSubscriberResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - - SimplePanel(str, "Info", [&response](IOutputStream& str) { - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDesc(str, "Owner", ActorIdFromProto(response.GetOwner())); - TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); - TermDesc(str, "DelayedSyncRequest", response.GetDelayedSyncRequest()); - TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); - TermDesc(str, "Path", GetPath(response)); - } - } - }); - - CollapsedPanel(str, "State", "state", [&response](IOutputStream& str) { - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDesc(str, "Deleted", response.GetState().GetDeleted()); - TermDesc(str, "Strong", response.GetState().GetStrong()); - TermDesc(str, "PathId", PathIdFromProto(response.GetState().GetPathId())); - TermDesc(str, "Version", response.GetState().GetVersion()); - TermDesc(str, "DomainId", PathIdFromProto(response.GetState().GetDomainId())); - TermDesc(str, "AbandonedSchemeShards", JoinSeq(", ", response.GetState().GetAbandonedSchemeShards())); - } - } - }); - - CollapsedPanel(str, "ProxyStates", "proxyStates", [&response](IOutputStream& str) { - HTML(str) { - TABLE_CLASS("table table-hover") { - TABLEHEAD() { - TABLER() { - TABLEH() { str << "#"; } - TABLEH() { str << "Actor"; } - TABLEH() { str << "Deleted"; } - TABLEH() { str << "Strong"; } - TABLEH() { str << "PathId"; } - TABLEH() { str << "Version"; } - TABLEH() { str << "DomainId"; } - TABLEH() { str << "AbandonedSchemeShards"; } - } - } - TABLEBODY() { - for (ui32 i = 0; i < response.ProxyStatesSize(); ++i) { - const auto& proxy = response.GetProxyStates(i); - const auto& state = proxy.GetState(); - - TABLER() { - TABLED() { str << (i + 1); } - TABLED() { Link(str, TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, proxy.GetProxy()); } - TABLED() { str << state.GetDeleted(); } - TABLED() { str << state.GetStrong(); } - TABLED() { str << PathIdFromProto(state.GetPathId()); } - TABLED() { str << state.GetVersion(); } - TABLED() { str << PathIdFromProto(state.GetDomainId()); } - TABLED() { str << JoinSeq(", ", state.GetAbandonedSchemeShards()); } - } - } - } - } - } - }); - } - - return str.Str(); - } - - static TString RenderSubscriberProxy(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberProxyResponse); - const auto& response = record.GetSubscriberProxyResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - - SimplePanel(str, "Info", [&response](IOutputStream& str) { - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, response.GetParent()); - TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); - TermDescLink(str, "ReplicaSubscriber", TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, response.GetReplicaSubscriber()); - TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); - TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); - TermDesc(str, "Path", GetPath(response)); - } - } - }); - } - - return str.Str(); - } - - static TString RenderReplicaSubscriber(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaSubscriberResponse); - const auto& response = record.GetReplicaSubscriberResponse(); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - - SimplePanel(str, "Info", [&response](IOutputStream& str) { - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, response.GetParent()); - TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); - TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); - TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); - TermDesc(str, "Path", GetPath(response)); - } - } - }); - } - - return str.Str(); - } - - static TString RenderCache(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { - Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kCacheResponse); - - TStringStream str; - - HTML(str) { - Header(str, record.GetActivityType(), record.GetSelf()); - - SimplePanel(str, "Descriptions", [&record](IOutputStream& str) { - const auto& response = record.GetCacheResponse(); - - HTML(str) { - DL_CLASS("dl-horizontal") { - TermDesc(str, "TotalCount", response.GetItemsTotalCount()); - TermDesc(str, "ByPathCount", response.GetItemsByPathCount()); - TermDesc(str, "ByPathIdCount", response.GetItemsByPathIdCount()); - } - - DescribeForm(str, ActorIdFromProto(record.GetSelf())); - } - }); - } - - return str.Str(); - } - - static TActorId MakeStateStorageProxyId() { - const auto& domains = AppData()->DomainsInfo->Domains; - Y_VERIFY(domains.size() <= 1); - - for (const auto& domain : domains) { - return NKikimr::MakeStateStorageProxyID(domain.second->DefaultSchemeBoardGroup); - } - - Y_FAIL("unreachable"); - } - - template <typename TDerived, typename TEvResponse> - class TBaseRequester: public TActorBootstrapped<TDerived> { - static constexpr char HTTPBADGATEWAY[] = "HTTP/1.1 502 Bad Gateway\r\nConnection: Close\r\n\r\nBad Gateway\r\n"; - static constexpr char HTTPUNAVAILABLE[] = "HTTP/1.1 503 Service Unavailable\r\nConnection: Close\r\n\r\nService Unavailable\r\n"; - static constexpr char HTTPTIMEOUT[] = "HTTP/1.1 504 Gateway Timeout\r\nConnection: Close\r\n\r\nGateway Timeout\r\n"; - - protected: - virtual IEventBase* MakeRequest() const = 0; - virtual void ProcessResponse(typename TEvResponse::TPtr& ev) = 0; - - void Handle(typename TEvResponse::TPtr& ev) { - ProcessResponse(ev); - } - - void Reply(const TString& content, EContentType type = EContentType::Html) { - this->Send(ReplyTo, new NMon::TEvHttpInfoRes(content, 0, type)); - this->PassAway(); - } - - void BadGateway() { - Reply(HTTPBADGATEWAY, EContentType::Custom); - } - - void Timeout() { - Reply(HTTPTIMEOUT, EContentType::Custom); - } - - void Unavailable() { - Reply(HTTPUNAVAILABLE, EContentType::Custom); - } - - public: + } + + default: + return EAttributeType::Unknown; + } + } + + template <typename P, typename D> + static bool TryGetNext(TStringBuf& s, D delim, P& param) { + TMaybe<TStringBuf> buf; + GetNext(s, delim, buf); + if (!buf) { + return false; + } + + return TryFromString(*buf, param); + } + + static TActorId ParseActorId(TStringBuf str) { + ui64 x1; + ui64 x2; + + if (!TryGetNext(str, ':', x1) || !TryGetNext(str, ':', x2)) { + return {}; + } + + return TActorId(x1, x2); + } + + static TActorId ParseActorIdFromPath(TStringBuf relPath) { + if (relPath.EndsWith('/')) { + relPath.Chop(1); + } + + auto lastPart = relPath.RNextTok('/'); + if (!lastPart) { + return {}; + } + + return ParseActorId(lastPart); + } + + static TActorId ParseActorId(const TJsonValue& value) { + if (value.GetType() != JSON_STRING) { + return {}; + } + + return ParseActorId(TStringBuf(value.GetStringSafe())); + } + + static EActivityType ParseActivityType(const TJsonValue& value) { + if (value.GetType() != JSON_STRING) { + return TActivity::OTHER; + } + + EActivityType result; + if (!TActivity::EType_Parse(value.GetStringSafe(), &result)) { + return TActivity::OTHER; + } + + return result; + } + + static std::pair<EActivityType, TActorId> ParseActorId(const TJsonValue::TMapType& map) { + const auto* activityType = map.FindPtr("ActivityType"); + const auto* actorId = map.FindPtr("ActorId"); + + if (!activityType || !actorId) { + return {}; + } + + return std::make_pair(ParseActivityType(*activityType), ParseActorId(*actorId)); + } + + static TPathId PathIdFromProto(const NKikimrSchemeBoardMon::TPathId& proto) { + return TPathId(proto.GetOwnerId(), proto.GetLocalPathId()); + } + + template <typename T> + static TString GetPath(const T& proto) { + if (proto.HasPath()) { + return proto.GetPath(); + } else if (proto.HasPathId()) { + return ToString(PathIdFromProto(proto.GetPathId())); + } + + return {}; + } + + const TJsonValue::TMapType& GetAttrs(const TActorId& actorId) const { + auto it = RegisteredActors.find(actorId); + Y_VERIFY(it != RegisteredActors.end()); + + return it->second.Attributes.GetMapSafe(); + } + + using TRenderer = std::function<void(IOutputStream&)>; + + template <typename T, typename U> + static void Header(IOutputStream& str, const T& title, const U& subTitile) { + HTML(str) { + DIV_CLASS("page-header") { + H3() { + str << title; + if (subTitile) { + SMALL() { str << " " << subTitile; } + } + } + } + } + } + + template <> + void Header(IOutputStream& str, const TString& activityType, const NActorsProto::TActorId& actorId) { + Header(str, activityType, ActorIdFromProto(actorId)); + } + + static void Alert(IOutputStream& str, const TStringBuf text) { + HTML(str) { + DIV_CLASS("alert alert-warning") { + STRONG() { + str << "Warning:"; + } + str << " " << text << "."; + } + } + } + + static void Panel(IOutputStream& str, TRenderer title, TRenderer body) { + HTML(str) { + DIV_CLASS("panel panel-default") { + DIV_CLASS("panel-heading") { + H4_CLASS("panel-title") { + title(str); + } + } + body(str); + } + } + } + + static void SimplePanel(IOutputStream& str, const TStringBuf title, TRenderer body) { + auto titleRenderer = [&title](IOutputStream& str) { + HTML(str) { + str << title; + } + }; + + auto bodyRenderer = [body = std::move(body)](IOutputStream& str) { + HTML(str) { + DIV_CLASS("panel-body") { + body(str); + } + } + }; + + Panel(str, titleRenderer, bodyRenderer); + } + + static void CollapsedPanel(IOutputStream& str, const TStringBuf title, const TStringBuf targetId, TRenderer body) { + auto titleRenderer = [&title, &targetId](IOutputStream& str) { + HTML(str) { + str << "<a data-toggle='collapse' href='#" << targetId << "'>" + << title + << "</a>"; + } + }; + + auto bodyRenderer = [&targetId, body = std::move(body)](IOutputStream& str) { + HTML(str) { + str << "<div id='" << targetId << "' class='collapse'>"; + DIV_CLASS("panel-body") { + body(str); + } + str << "</div>"; + } + }; + + Panel(str, titleRenderer, bodyRenderer); + } + + static ERequestType ConvertActivityType(EActivityType activityType) { + static THashMap<EActivityType, ERequestType> activityToRequest = { + {TActivity::SCHEME_BOARD_POPULATOR_ACTOR, ERequestType::Populator}, + {TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, ERequestType::ReplicaPopulator}, + {TActivity::SCHEME_BOARD_REPLICA_ACTOR, ERequestType::Replica}, + {TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, ERequestType::Subscriber}, + {TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, ERequestType::SubscriberProxy}, + {TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, ERequestType::ReplicaSubscriber}, + {TActivity::PROXY_SCHEME_CACHE, ERequestType::Cache}, + }; + + return activityToRequest.Value(activityType, ERequestType::Unknown); + } + + static TString ActorIdToStringSafe(const TActorId& actorId) { + return TStringBuilder() << actorId.RawX1() << ":" << actorId.RawX2(); + } + + template <typename T> + static void Link(IOutputStream& str, const TStringBuf path, const T& title) { + HTML(str) { + HREF(path) { + str << title; + } + } + } + + static void Link(IOutputStream& str, ERequestType requestType, const TStringBuf title, const TStringBuf prefix = "..") { + Link(str, MakeLink(requestType, prefix), title); + } + + static void Link(IOutputStream& str, EActivityType activityType, const TActorId& actorId, const TStringBuf prefix = "..") { + const TString path = TStringBuilder() + << MakeLink(ConvertActivityType(activityType), prefix) + << "/" << ActorIdToStringSafe(actorId); + Link(str, path, actorId); + } + + static void Link(IOutputStream& str, EActivityType activityType, const NActorsProto::TActorId& actorId, const TStringBuf prefix = "..") { + Link(str, activityType, ActorIdFromProto(actorId), prefix); + } + + template <typename T> + static void TermDesc(IOutputStream& str, const TStringBuf term, const T& desc) { + HTML(str) { + DT() { str << term; } + DD() { str << desc; } + } + } + + template <typename T> + static void TermDescLink(IOutputStream& str, const TStringBuf term, EActivityType activityType, const T& actorId) { + HTML(str) { + DT() { str << term; } + DD() { Link(str, activityType, actorId); } + } + } + + enum EFormType : ui8 { + ByPath = 1 << 0, + ByPathId = 1 << 1, + Both = ByPath | ByPathId, + }; + + static void Form(IOutputStream& str, EFormType formType, ERequestType linkType, const TStringBuf linkPrefix, const TActorId& actorId) { + HTML(str) { + FORM_CLASS("form-horizontal") { + const auto action = MakeLink(linkType, linkPrefix); + + if (formType & EFormType::ByPath) { + DIV_CLASS("form-group") { + LABEL_CLASS_FOR("col-sm-2 control-label", "path") { + str << "Path"; + } + DIV_CLASS("col-sm-8") { + str << "<input type='text' id='path' name='path' class='form-control' placeholder='/full/path'>"; + } + DIV_CLASS("col-sm-2") { + str << "<button type='submit' name='byPath' formaction='" << action << "' class='btn btn-primary'>" + << "Find by path" + << "</button>"; + } + } + } + + if (formType & EFormType::ByPathId) { + DIV_CLASS("form-group") { + LABEL_CLASS_FOR("col-sm-2 control-label", "pathId") { + str << "PathId"; + } + DIV_CLASS("col-sm-4") { + str << "<input type='number' id='ownerId' name='ownerId' class='form-control' placeholder='owner id'>"; + } + DIV_CLASS("col-sm-4") { + str << "<input type='number' id='pathId' name='pathId' class='form-control' placeholder='local path id'>"; + } + DIV_CLASS("col-sm-2") { + str << "<button type='submit' name='byPathId' formaction='" << action << "' class='btn btn-info'>" + << "Find by path id" + << "</button>"; + } + } + } + + if (actorId) { + str << "<input type='hidden' name='actorId' value='" << ActorIdToStringSafe(actorId) << "'>"; + } + + str << "<pre id='description' class='hidden'/>"; + + str << R"(<script> + $(document).ready(function() { + $('button').click(function(e) { + e.preventDefault(); + + var btn = this; + var form = $('form'); + + $.ajax({ + type: "GET", + url: btn.formAction, + data: Object.assign({[btn.name]: 1}, form.serializeArray().reduce(function(obj, cur, _) { + obj[cur.name] = cur.value; + return obj; + }, {})), + success: function(data) { + $('#description').text(JSON.stringify(data, null, ' ')).removeClass('hidden'); + }, + error: function (data) { + $('#description').text("Error: " + data).removeClass('hidden'); + }, + }); + }); + }); + </script>)"; + } + } + } + + static void ResolveForm(IOutputStream& str) { + Form(str, EFormType::Both, ERequestType::Resolve, "", {}); + } + + static void DescribeForm(IOutputStream& str, const TActorId& actorId, EFormType type = EFormType::Both) { + Form(str, type, ERequestType::Describe, "..", actorId); + } + + static void Navbar(IOutputStream& str, ERequestType originRequestType) { + static THashMap<ERequestType, TStringBuf> requestTypeToTitle = { + {ERequestType::Index, "Main"}, + {ERequestType::Resolver, "Resolver"}, + }; + + const bool isIndex = originRequestType == ERequestType::Index; + + HTML(str) { + TAG_CLASS(TNav, "navbar") { + UL_CLASS("nav nav-pills") { + for (const auto& [rt, title] : requestTypeToTitle) { + if (rt == originRequestType) { + LI_CLASS("active") { Link(str, "#", title); } + } else { + LI() { Link(str, rt, title, isIndex ? ROOT : ".."); } + } + } + } + } + } + } + + TString RenderIndex() const { + TStringStream str; + + HTML(str) { + Navbar(str, ERequestType::Index); + + for (const auto& kv : ByActivityType) { + const auto& activityType = kv.first; + const auto& actorIds = kv.second; + const auto activityTypeStr = TActivity::EType_Name(activityType); + + CollapsedPanel(str, activityTypeStr, activityTypeStr, [&](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + + Y_VERIFY(!actorIds.empty()); + for (const auto& [key, _] : GetAttrs(*actorIds.begin())) { + TABLEH() { str << key; } + } + } + } + TABLEBODY() { + int i = 1; + for (const auto& actorId : actorIds) { + TABLER() { + TABLED() { str << i++; } + TABLED() { Link(str, activityType, actorId, ROOT); } + + for (const auto& [_, value] : GetAttrs(actorId)) { + switch (ParseAttributeType(value)) { + case EAttributeType::Pod: + TABLED() { str << value; } + break; + + case EAttributeType::String: + TABLED() { str << value.GetStringSafe(); } + break; + + case EAttributeType::ActorId: { + const auto kv = ParseActorId(value.GetMapSafe()); + TABLED() { Link(str, kv.first, kv.second, ROOT); } + break; + } + + case EAttributeType::Unknown: + break; + } + } + } + } + } + } + } + }); + } + } + + return str.Str(); + } + + static TString RenderResolver(const TList<TActorId>& replicas) { + TStringStream str; + + HTML(str) { + Navbar(str, ERequestType::Resolver); + Header(str, "Replica resolver", ""); + + SimplePanel(str, "Resolver", [](IOutputStream& str) { + ResolveForm(str); + }); + + CollapsedPanel(str, "All replicas", "allReplicas", [&replicas](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + } + } + TABLEBODY() { + ui32 i = 0; + for (const auto& replica : replicas) { + TABLER() { + TABLED() { str << ++i; } + TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_ACTOR, replica, ""); } + } + } + } + } + } + }); + } + + return str.Str(); + } + + static TString RenderReplica(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaResponse); + const auto& response = record.GetReplicaResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + if (record.GetTruncated()) { + Alert(str, "some lists has been truncated"); + } + + SimplePanel(str, "Descriptions", [&record](IOutputStream& str) { + const auto& response = record.GetReplicaResponse(); + + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDesc(str, "TotalCount", response.GetDescriptions().GetTotalCount()); + TermDesc(str, "ByPathCount", response.GetDescriptions().GetByPathCount()); + TermDesc(str, "ByPathIdCount", response.GetDescriptions().GetByPathIdCount()); + } + + DescribeForm(str, ActorIdFromProto(record.GetSelf())); + } + }); + + CollapsedPanel(str, "Populators", "populators", [&response](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + TABLEH() { str << "Owner"; } + TABLEH() { str << "Generation"; } + TABLEH() { str << "PendingGeneration"; } + } + } + TABLEBODY() { + for (ui32 i = 0; i < response.PopulatorsSize(); ++i) { + const auto& populator = response.GetPopulators(i); + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, populator.GetActorId()); } + TABLED() { str << populator.GetOwner(); } + TABLED() { str << populator.GetGeneration(); } + TABLED() { str << populator.GetPendingGeneration(); } + } + } + } + } + } + }); + + CollapsedPanel(str, "Subscribers", "subscribers", [&response](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + TABLEH() { str << "Path"; } + } + } + TABLEBODY() { + for (ui32 i = 0; i < response.SubscribersSize(); ++i) { + const auto& subscriber = response.GetSubscribers(i); + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, subscriber.GetActorId()); } + TABLED() { str << GetPath(subscriber); } + } + } + } + } + } + }); + } + + return str.Str(); + } + + static TString RenderPopulator(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kPopulatorResponse); + const auto& response = record.GetPopulatorResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + if (record.GetTruncated()) { + Alert(str, "some lists has been truncated"); + } + + SimplePanel(str, "Info", [&record](IOutputStream& str) { + const auto& response = record.GetPopulatorResponse(); + + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDesc(str, "Owner", response.GetOwner()); + TermDesc(str, "Generation", response.GetGeneration()); + TermDesc(str, "DescriptionsCount", response.GetDescriptionsCount()); + TermDesc(str, "MaxPathId", PathIdFromProto(response.GetMaxPathId())); + TermDesc(str, "DelayedUpdatesCount", response.GetDelayedUpdatesCount()); + } + + DescribeForm(str, ActorIdFromProto(record.GetSelf()), EFormType::ByPathId); + } + }); + + CollapsedPanel(str, "ReplicaPopulators", "replicaPopulators", [&response](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + } + } + TABLEBODY() { + for (ui32 i = 0; i < response.ReplicaPopulatorsSize(); ++i) { + const auto& replicaPopulator = response.GetReplicaPopulators(i); + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { Link(str, TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR, replicaPopulator); } + } + } + } + } + } + }); + + CollapsedPanel(str, "UpdateAcks", "updateAcks", [&response](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Cookie"; } + TABLEH() { str << "AckTo"; } + TABLEH() { str << "PathId / Version / AcksCount"; } + } + } + TABLEBODY() { + for (ui32 i = 0; i < response.UpdateAcksSize(); ++i) { + const auto& updateAck = response.GetUpdateAcks(i); + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { str << updateAck.GetCookie(); } + TABLED() { str << ActorIdFromProto(updateAck.GetAckTo()); } + TABLED() { + TABLE_CLASS("table table-condensed") { + TABLEBODY() { + for (ui32 i = 0; i < updateAck.PathAcksSize(); ++i) { + const auto& pathAck = updateAck.GetPathAcks(i); + + TABLER() { + TABLED() { str << PathIdFromProto(pathAck.GetPathId()); } + TABLED() { str << pathAck.GetVersion(); } + TABLED() { str << pathAck.GetAcksCount(); } + } + } + } + } + } + } + } + } + } + } + }); + } + + return str.Str(); + } + + static TString RenderReplicaPopulator(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaPopulatorResponse); + const auto& response = record.GetReplicaPopulatorResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + if (record.GetTruncated()) { + Alert(str, "some lists has been truncated"); + } + + SimplePanel(str, "Info", [&response](IOutputStream& str) { + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_POPULATOR_ACTOR, response.GetParent()); + TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); + TermDesc(str, "Owner", response.GetOwner()); + TermDesc(str, "Generation", response.GetGeneration()); + TermDesc(str, "CurPathId", PathIdFromProto(response.GetCurPathId())); + TermDesc(str, "LastAckedPathId", PathIdFromProto(response.GetLastAckedPathId())); + TermDesc(str, "BatchSize", response.GetBatchSize()); + TermDesc(str, "BatchSizeLimit", response.GetBatchSizeLimit()); + } + } + }); + + auto renderUpdates = [&str](const TStringBuf title, const TStringBuf targetId, const auto& updates) { + CollapsedPanel(str, title, targetId, [&updates](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "PathId"; } + TABLEH() { str << "Version / TxIds"; } + } + } + TABLEBODY() { + for (int i = 0; i < updates.size(); ++i) { + const auto& update = updates[i]; + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { str << PathIdFromProto(update.GetPathId()); } + TABLED() { + TABLE_CLASS("table table-condensed") { + TABLEBODY() { + for (ui32 i = 0; i < update.VersionsSize(); ++i) { + const auto& version = update.GetVersions(i); + + TABLER() { + TABLED() { str << version.GetVersion(); } + TABLED() { str << JoinSeq(", ", version.GetTxIds()); } + } + } + } + } + } + } + } + } + } + } + }); + }; + + renderUpdates("Updates", "updates", response.GetUpdates()); + renderUpdates("UpdatesRequested", "updatesRequested", response.GetUpdatesRequested()); + renderUpdates("UpdatesInFlight", "updatesInFlight", response.GetUpdatesInFlight()); + } + + return str.Str(); + } + + static TString RenderSubscriber(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberResponse); + const auto& response = record.GetSubscriberResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + + SimplePanel(str, "Info", [&response](IOutputStream& str) { + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDesc(str, "Owner", ActorIdFromProto(response.GetOwner())); + TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); + TermDesc(str, "DelayedSyncRequest", response.GetDelayedSyncRequest()); + TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); + TermDesc(str, "Path", GetPath(response)); + } + } + }); + + CollapsedPanel(str, "State", "state", [&response](IOutputStream& str) { + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDesc(str, "Deleted", response.GetState().GetDeleted()); + TermDesc(str, "Strong", response.GetState().GetStrong()); + TermDesc(str, "PathId", PathIdFromProto(response.GetState().GetPathId())); + TermDesc(str, "Version", response.GetState().GetVersion()); + TermDesc(str, "DomainId", PathIdFromProto(response.GetState().GetDomainId())); + TermDesc(str, "AbandonedSchemeShards", JoinSeq(", ", response.GetState().GetAbandonedSchemeShards())); + } + } + }); + + CollapsedPanel(str, "ProxyStates", "proxyStates", [&response](IOutputStream& str) { + HTML(str) { + TABLE_CLASS("table table-hover") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "#"; } + TABLEH() { str << "Actor"; } + TABLEH() { str << "Deleted"; } + TABLEH() { str << "Strong"; } + TABLEH() { str << "PathId"; } + TABLEH() { str << "Version"; } + TABLEH() { str << "DomainId"; } + TABLEH() { str << "AbandonedSchemeShards"; } + } + } + TABLEBODY() { + for (ui32 i = 0; i < response.ProxyStatesSize(); ++i) { + const auto& proxy = response.GetProxyStates(i); + const auto& state = proxy.GetState(); + + TABLER() { + TABLED() { str << (i + 1); } + TABLED() { Link(str, TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, proxy.GetProxy()); } + TABLED() { str << state.GetDeleted(); } + TABLED() { str << state.GetStrong(); } + TABLED() { str << PathIdFromProto(state.GetPathId()); } + TABLED() { str << state.GetVersion(); } + TABLED() { str << PathIdFromProto(state.GetDomainId()); } + TABLED() { str << JoinSeq(", ", state.GetAbandonedSchemeShards()); } + } + } + } + } + } + }); + } + + return str.Str(); + } + + static TString RenderSubscriberProxy(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberProxyResponse); + const auto& response = record.GetSubscriberProxyResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + + SimplePanel(str, "Info", [&response](IOutputStream& str) { + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, response.GetParent()); + TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); + TermDescLink(str, "ReplicaSubscriber", TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR, response.GetReplicaSubscriber()); + TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); + TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); + TermDesc(str, "Path", GetPath(response)); + } + } + }); + } + + return str.Str(); + } + + static TString RenderReplicaSubscriber(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaSubscriberResponse); + const auto& response = record.GetReplicaSubscriberResponse(); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + + SimplePanel(str, "Info", [&response](IOutputStream& str) { + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDescLink(str, "Parent", TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, response.GetParent()); + TermDescLink(str, "Replica", TActivity::SCHEME_BOARD_REPLICA_ACTOR, response.GetReplica()); + TermDesc(str, "DomainOwnerId", response.GetDomainOwnerId()); + TermDesc(str, "CurrentSyncRequest", response.GetCurrentSyncRequest()); + TermDesc(str, "Path", GetPath(response)); + } + } + }); + } + + return str.Str(); + } + + static TString RenderCache(const NKikimrSchemeBoardMon::TEvInfoResponse& record) { + Y_VERIFY(record.GetResponseCase() == NKikimrSchemeBoardMon::TEvInfoResponse::kCacheResponse); + + TStringStream str; + + HTML(str) { + Header(str, record.GetActivityType(), record.GetSelf()); + + SimplePanel(str, "Descriptions", [&record](IOutputStream& str) { + const auto& response = record.GetCacheResponse(); + + HTML(str) { + DL_CLASS("dl-horizontal") { + TermDesc(str, "TotalCount", response.GetItemsTotalCount()); + TermDesc(str, "ByPathCount", response.GetItemsByPathCount()); + TermDesc(str, "ByPathIdCount", response.GetItemsByPathIdCount()); + } + + DescribeForm(str, ActorIdFromProto(record.GetSelf())); + } + }); + } + + return str.Str(); + } + + static TActorId MakeStateStorageProxyId() { + const auto& domains = AppData()->DomainsInfo->Domains; + Y_VERIFY(domains.size() <= 1); + + for (const auto& domain : domains) { + return NKikimr::MakeStateStorageProxyID(domain.second->DefaultSchemeBoardGroup); + } + + Y_FAIL("unreachable"); + } + + template <typename TDerived, typename TEvResponse> + class TBaseRequester: public TActorBootstrapped<TDerived> { + static constexpr char HTTPBADGATEWAY[] = "HTTP/1.1 502 Bad Gateway\r\nConnection: Close\r\n\r\nBad Gateway\r\n"; + static constexpr char HTTPUNAVAILABLE[] = "HTTP/1.1 503 Service Unavailable\r\nConnection: Close\r\n\r\nService Unavailable\r\n"; + static constexpr char HTTPTIMEOUT[] = "HTTP/1.1 504 Gateway Timeout\r\nConnection: Close\r\n\r\nGateway Timeout\r\n"; + + protected: + virtual IEventBase* MakeRequest() const = 0; + virtual void ProcessResponse(typename TEvResponse::TPtr& ev) = 0; + + void Handle(typename TEvResponse::TPtr& ev) { + ProcessResponse(ev); + } + + void Reply(const TString& content, EContentType type = EContentType::Html) { + this->Send(ReplyTo, new NMon::TEvHttpInfoRes(content, 0, type)); + this->PassAway(); + } + + void BadGateway() { + Reply(HTTPBADGATEWAY, EContentType::Custom); + } + + void Timeout() { + Reply(HTTPTIMEOUT, EContentType::Custom); + } + + void Unavailable() { + Reply(HTTPUNAVAILABLE, EContentType::Custom); + } + + public: static constexpr auto ActorActivityType() { - return TActivity::SCHEME_BOARD_INFO_REQUESTER_ACTOR; - } - - explicit TBaseRequester(const TActorId& requestFrom, const TActorId& replyTo) - : RequestFrom(requestFrom) - , ReplyTo(replyTo) - { - } - - void Bootstrap() { - this->Send(RequestFrom, MakeRequest(), IEventHandle::FlagTrackDelivery); - this->Become(&TDerived::StateWork, TDuration::Seconds(10), new TEvents::TEvWakeup()); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvResponse, Handle); - cFunc(TEvents::TEvWakeup::EventType, Timeout); - cFunc(TEvents::TEvUndelivered::EventType, Unavailable); - } - } - - using TBase = TBaseRequester<TDerived, TEvResponse>; - - private: - const TActorId RequestFrom; - const TActorId ReplyTo; - - }; // TBaseRequester - - class TReplicaEnumerator: public TBaseRequester<TReplicaEnumerator, TEvStateStorage::TEvListSchemeBoardResult> { - IEventBase* MakeRequest() const override { - return new TEvStateStorage::TEvListSchemeBoard(); - } - - void ProcessResponse(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) override { - const auto& info = ev->Get()->Info; - if (!info) { - return BadGateway(); - } - - return Reply(RenderResolver(info->SelectAllReplicas())); - } - - public: - explicit TReplicaEnumerator(const TActorId& replyTo) - : TBase(MakeStateStorageProxyId(), replyTo) - { - } - - }; // TReplicaEnumerator - - class TReplicaResolver: public TBaseRequester<TReplicaResolver, TEvStateStorage::TEvResolveReplicasList> { - IEventBase* MakeRequest() const override { - switch (Path.index()) { - case 0: - return new TEvStateStorage::TEvResolveSchemeBoard(std::get<TString>(Path)); - case 1: - return new TEvStateStorage::TEvResolveSchemeBoard(std::get<TPathId>(Path)); - default: - Y_FAIL("unreachable"); - } - } - - void ProcessResponse(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) override { - TJsonValue json; - - auto& replicas = json["replicas"]; - replicas.SetType(JSON_ARRAY); - - for (const auto& replica : ev->Get()->Replicas) { - replicas.AppendValue(ToString(replica)); - } - - Reply(TStringBuilder() << NMonitoring::HTTPOKJSON << WriteJson(json), EContentType::Custom); - } - - public: - template <typename T> - explicit TReplicaResolver(const TActorId& requestFrom, const TActorId& replyTo, const T& path) - : TBase(requestFrom, replyTo) - , Path(path) - { - } - - private: - std::variant<TString, TPathId> Path; - - }; // TReplicaResolver - - class TInfoRequester: public TBaseRequester<TInfoRequester, TSchemeBoardMonEvents::TEvInfoResponse> { - IEventBase* MakeRequest() const override { - return new TSchemeBoardMonEvents::TEvInfoRequest(); - } - - void ProcessResponse(TSchemeBoardMonEvents::TEvInfoResponse::TPtr& ev) override { - const auto& record = ev->Get()->Record; - - switch (record.GetResponseCase()) { - case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaResponse: - return Reply(RenderReplica(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kPopulatorResponse: - return Reply(RenderPopulator(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaPopulatorResponse: - return Reply(RenderReplicaPopulator(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberResponse: - return Reply(RenderSubscriber(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberProxyResponse: - return Reply(RenderSubscriberProxy(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaSubscriberResponse: - return Reply(RenderReplicaSubscriber(record)); - case NKikimrSchemeBoardMon::TEvInfoResponse::kCacheResponse: - return Reply(RenderCache(record)); - default: - return BadGateway(); - } - } - - public: - using TBase::TBase; - - }; // TInfoRequester - - class TDescriber: public TBaseRequester<TDescriber, TSchemeBoardMonEvents::TEvDescribeResponse> { - IEventBase* MakeRequest() const override { - switch (Path.index()) { - case 0: + return TActivity::SCHEME_BOARD_INFO_REQUESTER_ACTOR; + } + + explicit TBaseRequester(const TActorId& requestFrom, const TActorId& replyTo) + : RequestFrom(requestFrom) + , ReplyTo(replyTo) + { + } + + void Bootstrap() { + this->Send(RequestFrom, MakeRequest(), IEventHandle::FlagTrackDelivery); + this->Become(&TDerived::StateWork, TDuration::Seconds(10), new TEvents::TEvWakeup()); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvResponse, Handle); + cFunc(TEvents::TEvWakeup::EventType, Timeout); + cFunc(TEvents::TEvUndelivered::EventType, Unavailable); + } + } + + using TBase = TBaseRequester<TDerived, TEvResponse>; + + private: + const TActorId RequestFrom; + const TActorId ReplyTo; + + }; // TBaseRequester + + class TReplicaEnumerator: public TBaseRequester<TReplicaEnumerator, TEvStateStorage::TEvListSchemeBoardResult> { + IEventBase* MakeRequest() const override { + return new TEvStateStorage::TEvListSchemeBoard(); + } + + void ProcessResponse(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) override { + const auto& info = ev->Get()->Info; + if (!info) { + return BadGateway(); + } + + return Reply(RenderResolver(info->SelectAllReplicas())); + } + + public: + explicit TReplicaEnumerator(const TActorId& replyTo) + : TBase(MakeStateStorageProxyId(), replyTo) + { + } + + }; // TReplicaEnumerator + + class TReplicaResolver: public TBaseRequester<TReplicaResolver, TEvStateStorage::TEvResolveReplicasList> { + IEventBase* MakeRequest() const override { + switch (Path.index()) { + case 0: + return new TEvStateStorage::TEvResolveSchemeBoard(std::get<TString>(Path)); + case 1: + return new TEvStateStorage::TEvResolveSchemeBoard(std::get<TPathId>(Path)); + default: + Y_FAIL("unreachable"); + } + } + + void ProcessResponse(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) override { + TJsonValue json; + + auto& replicas = json["replicas"]; + replicas.SetType(JSON_ARRAY); + + for (const auto& replica : ev->Get()->Replicas) { + replicas.AppendValue(ToString(replica)); + } + + Reply(TStringBuilder() << NMonitoring::HTTPOKJSON << WriteJson(json), EContentType::Custom); + } + + public: + template <typename T> + explicit TReplicaResolver(const TActorId& requestFrom, const TActorId& replyTo, const T& path) + : TBase(requestFrom, replyTo) + , Path(path) + { + } + + private: + std::variant<TString, TPathId> Path; + + }; // TReplicaResolver + + class TInfoRequester: public TBaseRequester<TInfoRequester, TSchemeBoardMonEvents::TEvInfoResponse> { + IEventBase* MakeRequest() const override { + return new TSchemeBoardMonEvents::TEvInfoRequest(); + } + + void ProcessResponse(TSchemeBoardMonEvents::TEvInfoResponse::TPtr& ev) override { + const auto& record = ev->Get()->Record; + + switch (record.GetResponseCase()) { + case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaResponse: + return Reply(RenderReplica(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kPopulatorResponse: + return Reply(RenderPopulator(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaPopulatorResponse: + return Reply(RenderReplicaPopulator(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberResponse: + return Reply(RenderSubscriber(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kSubscriberProxyResponse: + return Reply(RenderSubscriberProxy(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kReplicaSubscriberResponse: + return Reply(RenderReplicaSubscriber(record)); + case NKikimrSchemeBoardMon::TEvInfoResponse::kCacheResponse: + return Reply(RenderCache(record)); + default: + return BadGateway(); + } + } + + public: + using TBase::TBase; + + }; // TInfoRequester + + class TDescriber: public TBaseRequester<TDescriber, TSchemeBoardMonEvents::TEvDescribeResponse> { + IEventBase* MakeRequest() const override { + switch (Path.index()) { + case 0: return new TSchemeBoardMonEvents::TEvDescribeRequest(std::get<TString>(Path)); - case 1: + case 1: return new TSchemeBoardMonEvents::TEvDescribeRequest(std::get<TPathId>(Path)); - default: - Y_FAIL("unreachable"); - } - } - - void ProcessResponse(TSchemeBoardMonEvents::TEvDescribeResponse::TPtr& ev) override { - const auto& record = ev->Get()->Record; - Reply(TStringBuilder() << NMonitoring::HTTPOKJSON << record.GetJson(), EContentType::Custom); - } - - public: - template <typename T> - explicit TDescriber(const TActorId& requestFrom, const TActorId& replyTo, const T& path) - : TBase(requestFrom, replyTo) - , Path(path) - { - } - - private: + default: + Y_FAIL("unreachable"); + } + } + + void ProcessResponse(TSchemeBoardMonEvents::TEvDescribeResponse::TPtr& ev) override { + const auto& record = ev->Get()->Record; + Reply(TStringBuilder() << NMonitoring::HTTPOKJSON << record.GetJson(), EContentType::Custom); + } + + public: + template <typename T> + explicit TDescriber(const TActorId& requestFrom, const TActorId& replyTo, const T& path) + : TBase(requestFrom, replyTo) + , Path(path) + { + } + + private: std::variant<TString, TPathId> Path; - - }; // TDescriber - - template <typename TActionActor> - bool RunFormAction(const TActorId& requestFrom, const TActorId& replyTo, const TCgiParameters& params) { - if (params.Has("byPath")) { - Register(new TActionActor(requestFrom, replyTo, params.Get("path"))); - return true; - } else if (params.Has("byPathId")) { - ui64 oid; - ui64 lpid; - if (TryFromString(params.Get("ownerId"), oid) && TryFromString(params.Get("pathId"), lpid)) { - Register(new TActionActor(requestFrom, replyTo, TPathId(oid, lpid))); - return true; - } - } - - return false; - } - - void Handle(NMon::TEvHttpInfo::TPtr& ev) { - const auto& request = ev->Get()->Request; - const auto& params = request.GetParams(); - - switch (ParseRequestType(request.GetPathInfo())) { - case ERequestType::Index: - return (void)Send(ev->Sender, new NMon::TEvHttpInfoRes(RenderIndex())); - - case ERequestType::Populator: - case ERequestType::ReplicaPopulator: - case ERequestType::Replica: - case ERequestType::Subscriber: - case ERequestType::SubscriberProxy: - case ERequestType::ReplicaSubscriber: - case ERequestType::Cache: - if (const auto actorId = ParseActorIdFromPath(request.GetPathInfo())) { - return (void)Register(new TInfoRequester(actorId, ev->Sender)); - } - break; - - case ERequestType::Describe: - if (const auto actorId = ParseActorId(TStringBuf(params.Get("actorId")))) { - if (RunFormAction<TDescriber>(actorId, ev->Sender, params)) { - return; - } - } - break; - - case ERequestType::Resolver: - return (void)Register(new TReplicaEnumerator(ev->Sender)); - - case ERequestType::Resolve: - if (RunFormAction<TReplicaResolver>(MakeStateStorageProxyId(), ev->Sender, params)) { - return; - } - break; - - case ERequestType::Unknown: - break; - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(NMonitoring::HTTPNOTFOUND, 0, EContentType::Custom)); - } - -public: + + }; // TDescriber + + template <typename TActionActor> + bool RunFormAction(const TActorId& requestFrom, const TActorId& replyTo, const TCgiParameters& params) { + if (params.Has("byPath")) { + Register(new TActionActor(requestFrom, replyTo, params.Get("path"))); + return true; + } else if (params.Has("byPathId")) { + ui64 oid; + ui64 lpid; + if (TryFromString(params.Get("ownerId"), oid) && TryFromString(params.Get("pathId"), lpid)) { + Register(new TActionActor(requestFrom, replyTo, TPathId(oid, lpid))); + return true; + } + } + + return false; + } + + void Handle(NMon::TEvHttpInfo::TPtr& ev) { + const auto& request = ev->Get()->Request; + const auto& params = request.GetParams(); + + switch (ParseRequestType(request.GetPathInfo())) { + case ERequestType::Index: + return (void)Send(ev->Sender, new NMon::TEvHttpInfoRes(RenderIndex())); + + case ERequestType::Populator: + case ERequestType::ReplicaPopulator: + case ERequestType::Replica: + case ERequestType::Subscriber: + case ERequestType::SubscriberProxy: + case ERequestType::ReplicaSubscriber: + case ERequestType::Cache: + if (const auto actorId = ParseActorIdFromPath(request.GetPathInfo())) { + return (void)Register(new TInfoRequester(actorId, ev->Sender)); + } + break; + + case ERequestType::Describe: + if (const auto actorId = ParseActorId(TStringBuf(params.Get("actorId")))) { + if (RunFormAction<TDescriber>(actorId, ev->Sender, params)) { + return; + } + } + break; + + case ERequestType::Resolver: + return (void)Register(new TReplicaEnumerator(ev->Sender)); + + case ERequestType::Resolve: + if (RunFormAction<TReplicaResolver>(MakeStateStorageProxyId(), ev->Sender, params)) { + return; + } + break; + + case ERequestType::Unknown: + break; + } + + Send(ev->Sender, new NMon::TEvHttpInfoRes(NMonitoring::HTTPNOTFOUND, 0, EContentType::Custom)); + } + +public: static constexpr auto ActorActivityType() { - return TActivity::SCHEME_BOARD_MONITORING_ACTOR; - } - - void Bootstrap() { - if (auto* mon = AppData()->Mon) { - auto* actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); - mon->RegisterActorPage(actorsMonPage, ROOT, "Scheme Board", - false, TlsActivationContext->ActorSystem(), SelfId()); - } - - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardMonEvents::TEvRegister, Handle); - hFunc(TSchemeBoardMonEvents::TEvUnregister, Handle); - - hFunc(NMon::TEvHttpInfo, Handle); - - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - -private: - THashMap<TActorId, TActorInfo> RegisteredActors; - THashMap<EActivityType, THashSet<TActorId>> ByActivityType; - -}; // TMonitoring - -} // NSchemeBoard - -IActor* CreateSchemeBoardMonitoring() { - return new NSchemeBoard::TMonitoring(); -} - -} // NKikimr + return TActivity::SCHEME_BOARD_MONITORING_ACTOR; + } + + void Bootstrap() { + if (auto* mon = AppData()->Mon) { + auto* actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); + mon->RegisterActorPage(actorsMonPage, ROOT, "Scheme Board", + false, TlsActivationContext->ActorSystem(), SelfId()); + } + + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardMonEvents::TEvRegister, Handle); + hFunc(TSchemeBoardMonEvents::TEvUnregister, Handle); + + hFunc(NMon::TEvHttpInfo, Handle); + + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + +private: + THashMap<TActorId, TActorInfo> RegisteredActors; + THashMap<EActivityType, THashSet<TActorId>> ByActivityType; + +}; // TMonitoring + +} // NSchemeBoard + +IActor* CreateSchemeBoardMonitoring() { + return new NSchemeBoard::TMonitoring(); +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/monitoring.h b/ydb/core/tx/scheme_board/monitoring.h index 92154728063..6bbd25885c3 100644 --- a/ydb/core/tx/scheme_board/monitoring.h +++ b/ydb/core/tx/scheme_board/monitoring.h @@ -1,14 +1,14 @@ -#pragma once - -#include "defs.h" - -namespace NKikimr { - -inline TActorId MakeSchemeBoardMonitoringId() { - char x[12] = { 's', 'b', 'm', 'o', 'n' }; - return TActorId(0, TStringBuf(x, 12)); -} - -IActor* CreateSchemeBoardMonitoring(); - -} // NKikimr +#pragma once + +#include "defs.h" + +namespace NKikimr { + +inline TActorId MakeSchemeBoardMonitoringId() { + char x[12] = { 's', 'b', 'm', 'o', 'n' }; + return TActorId(0, TStringBuf(x, 12)); +} + +IActor* CreateSchemeBoardMonitoring(); + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/monitoring_ut.cpp b/ydb/core/tx/scheme_board/monitoring_ut.cpp index 79d43ae9d7c..733dbc0ddb2 100644 --- a/ydb/core/tx/scheme_board/monitoring_ut.cpp +++ b/ydb/core/tx/scheme_board/monitoring_ut.cpp @@ -1,130 +1,130 @@ -#include "mon_events.h" -#include "monitoring.h" -#include "ut_helpers.h" - +#include "mon_events.h" +#include "monitoring.h" +#include "ut_helpers.h" + #include <ydb/core/testlib/basics/appdata.h> - -#include <library/cpp/actors/core/mon.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <util/string/printf.h> - -namespace NKikimr { -namespace NSchemeBoard { - -Y_UNIT_TEST_SUITE(TMonitoringTests) { - static constexpr char HTTPUNAVAILABLE[] = "HTTP/1.1 503 Service Unavailable\r\nConnection: Close\r\n\r\nService Unavailable\r\n"; - - struct TMockMonRequest: public NMonitoring::IMonHttpRequest { - explicit TMockMonRequest(const TString& pathInfo = {}, const TCgiParameters& params = {}) - : PathInfo(pathInfo) - , Params(params) - { - } - - TStringBuf GetPathInfo() const override { - return PathInfo; - } - - const TCgiParameters& GetParams() const override { - return Params; - } - - IOutputStream& Output() override { Y_FAIL("Not implemented"); } - HTTP_METHOD GetMethod() const override { Y_FAIL("Not implemented"); } - TStringBuf GetPath() const override { Y_FAIL("Not implemented"); } - TStringBuf GetUri() const override { Y_FAIL("Not implemented"); } - const TCgiParameters& GetPostParams() const override { Y_FAIL("Not implemented"); } - TStringBuf GetPostContent() const override { Y_FAIL("Not implemented"); } - const THttpHeaders& GetHeaders() const override { Y_FAIL("Not implemented"); } - TStringBuf GetHeader(TStringBuf) const override { Y_FAIL("Not implemented"); } - TStringBuf GetCookie(TStringBuf) const override { Y_FAIL("Not implemented"); } - TString GetRemoteAddr() const override { Y_FAIL("Not implemented"); } - TString GetServiceTitle() const override { Y_FAIL("Not implemented"); } - NMonitoring::IMonPage* GetPage() const override { Y_FAIL("Not implemented"); } - NMonitoring::IMonHttpRequest* MakeChild(NMonitoring::IMonPage*, const TString&) const override { Y_FAIL("Not implemented"); } - - private: - const TString PathInfo; - const TCgiParameters Params; - }; - - TActorId CreateMonitoring(TTestActorRuntimeBase& runtime) { - const auto actorId = runtime.Register(CreateSchemeBoardMonitoring()); - - // wait until actor is ready - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&actorId](IEventHandle& ev) { - return ev.Recipient == actorId && ev.GetTypeRewrite() == TEvents::TSystem::Bootstrap; - }); - runtime.DispatchEvents(opts); - - return actorId; - } - - void TestActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, - const TString& path, const TCgiParameters& params, const TString& expectedAnswer) - { - auto request = MakeHolder<TMockMonRequest>(path, params); - ctx.Send(monitoring, sender, new NMon::TEvHttpInfo(*request.Get())); - - auto ev = ctx.GrabEdgeEvent<NMon::TEvHttpInfoRes>(sender); - UNIT_ASSERT(ev->Get()); - - auto response = static_cast<NMon::TEvHttpInfoRes*>(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(response->Answer, expectedAnswer); - } - - void TestValidActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, - const TString& path, const TCgiParameters& params = {}) - { - TestActorId(ctx, monitoring, sender, path, params, HTTPUNAVAILABLE); - } - - Y_UNIT_TEST(ValidActorId) { - auto context = MakeHolder<TTestContext>(); - context->Initialize(TAppPrepare().Unwrap()); - - const auto monitoring = CreateMonitoring(*context); - const auto sender = context->AllocateEdgeActor(); - - for (const TString& actorId : {"7007306538023635091:201863463214"}) { - TestValidActorId(*context, monitoring, sender, Sprintf("/populator/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/replica_populator/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/replica/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/subscriber/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/subscriber_proxy/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/replica_subscriber/%s", actorId.data())); - TestValidActorId(*context, monitoring, sender, Sprintf("/cache/%s", actorId.data())); - TestActorId(*context, monitoring, sender, "/describe", {{"actorId", actorId}}, NMonitoring::HTTPNOTFOUND); - } - } - - void TestInvalidActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, - const TString& path, const TCgiParameters& params = {}) - { - TestActorId(ctx, monitoring, sender, path, params, NMonitoring::HTTPNOTFOUND); - } - - Y_UNIT_TEST(InvalidActorId) { - auto context = MakeHolder<TTestContext>(); - context->Initialize(TAppPrepare().Unwrap()); - - const auto monitoring = CreateMonitoring(*context); - const auto sender = context->AllocateEdgeActor(); - - for (const TString& actorId : {"Invalid", "Invalid:1", "1:Invalid"}) { - TestInvalidActorId(*context, monitoring, sender, Sprintf("/populator/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica_populator/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/subscriber/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/subscriber_proxy/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica_subscriber/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, Sprintf("/cache/%s", actorId.data())); - TestInvalidActorId(*context, monitoring, sender, "/describe", {{"actorId", actorId}}); - } - } -} - -} // NSchemeBoard -} // NKikimr + +#include <library/cpp/actors/core/mon.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/printf.h> + +namespace NKikimr { +namespace NSchemeBoard { + +Y_UNIT_TEST_SUITE(TMonitoringTests) { + static constexpr char HTTPUNAVAILABLE[] = "HTTP/1.1 503 Service Unavailable\r\nConnection: Close\r\n\r\nService Unavailable\r\n"; + + struct TMockMonRequest: public NMonitoring::IMonHttpRequest { + explicit TMockMonRequest(const TString& pathInfo = {}, const TCgiParameters& params = {}) + : PathInfo(pathInfo) + , Params(params) + { + } + + TStringBuf GetPathInfo() const override { + return PathInfo; + } + + const TCgiParameters& GetParams() const override { + return Params; + } + + IOutputStream& Output() override { Y_FAIL("Not implemented"); } + HTTP_METHOD GetMethod() const override { Y_FAIL("Not implemented"); } + TStringBuf GetPath() const override { Y_FAIL("Not implemented"); } + TStringBuf GetUri() const override { Y_FAIL("Not implemented"); } + const TCgiParameters& GetPostParams() const override { Y_FAIL("Not implemented"); } + TStringBuf GetPostContent() const override { Y_FAIL("Not implemented"); } + const THttpHeaders& GetHeaders() const override { Y_FAIL("Not implemented"); } + TStringBuf GetHeader(TStringBuf) const override { Y_FAIL("Not implemented"); } + TStringBuf GetCookie(TStringBuf) const override { Y_FAIL("Not implemented"); } + TString GetRemoteAddr() const override { Y_FAIL("Not implemented"); } + TString GetServiceTitle() const override { Y_FAIL("Not implemented"); } + NMonitoring::IMonPage* GetPage() const override { Y_FAIL("Not implemented"); } + NMonitoring::IMonHttpRequest* MakeChild(NMonitoring::IMonPage*, const TString&) const override { Y_FAIL("Not implemented"); } + + private: + const TString PathInfo; + const TCgiParameters Params; + }; + + TActorId CreateMonitoring(TTestActorRuntimeBase& runtime) { + const auto actorId = runtime.Register(CreateSchemeBoardMonitoring()); + + // wait until actor is ready + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&actorId](IEventHandle& ev) { + return ev.Recipient == actorId && ev.GetTypeRewrite() == TEvents::TSystem::Bootstrap; + }); + runtime.DispatchEvents(opts); + + return actorId; + } + + void TestActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, + const TString& path, const TCgiParameters& params, const TString& expectedAnswer) + { + auto request = MakeHolder<TMockMonRequest>(path, params); + ctx.Send(monitoring, sender, new NMon::TEvHttpInfo(*request.Get())); + + auto ev = ctx.GrabEdgeEvent<NMon::TEvHttpInfoRes>(sender); + UNIT_ASSERT(ev->Get()); + + auto response = static_cast<NMon::TEvHttpInfoRes*>(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(response->Answer, expectedAnswer); + } + + void TestValidActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, + const TString& path, const TCgiParameters& params = {}) + { + TestActorId(ctx, monitoring, sender, path, params, HTTPUNAVAILABLE); + } + + Y_UNIT_TEST(ValidActorId) { + auto context = MakeHolder<TTestContext>(); + context->Initialize(TAppPrepare().Unwrap()); + + const auto monitoring = CreateMonitoring(*context); + const auto sender = context->AllocateEdgeActor(); + + for (const TString& actorId : {"7007306538023635091:201863463214"}) { + TestValidActorId(*context, monitoring, sender, Sprintf("/populator/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/replica_populator/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/replica/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/subscriber/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/subscriber_proxy/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/replica_subscriber/%s", actorId.data())); + TestValidActorId(*context, monitoring, sender, Sprintf("/cache/%s", actorId.data())); + TestActorId(*context, monitoring, sender, "/describe", {{"actorId", actorId}}, NMonitoring::HTTPNOTFOUND); + } + } + + void TestInvalidActorId(TTestContext& ctx, const TActorId& monitoring, const TActorId& sender, + const TString& path, const TCgiParameters& params = {}) + { + TestActorId(ctx, monitoring, sender, path, params, NMonitoring::HTTPNOTFOUND); + } + + Y_UNIT_TEST(InvalidActorId) { + auto context = MakeHolder<TTestContext>(); + context->Initialize(TAppPrepare().Unwrap()); + + const auto monitoring = CreateMonitoring(*context); + const auto sender = context->AllocateEdgeActor(); + + for (const TString& actorId : {"Invalid", "Invalid:1", "1:Invalid"}) { + TestInvalidActorId(*context, monitoring, sender, Sprintf("/populator/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica_populator/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/subscriber/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/subscriber_proxy/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/replica_subscriber/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, Sprintf("/cache/%s", actorId.data())); + TestInvalidActorId(*context, monitoring, sender, "/describe", {{"actorId", actorId}}); + } + } +} + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/populator.cpp b/ydb/core/tx/scheme_board/populator.cpp index 6fcccdaf310..f860bb88cb1 100644 --- a/ydb/core/tx/scheme_board/populator.cpp +++ b/ydb/core/tx/scheme_board/populator.cpp @@ -1,103 +1,103 @@ -#include "events.h" -#include "helpers.h" -#include "monitorable_actor.h" -#include "populator.h" - -#include <contrib/libs/protobuf/src/google/protobuf/util/json_util.h> - +#include "events.h" +#include "helpers.h" +#include "monitorable_actor.h" +#include "populator.h" + +#include <contrib/libs/protobuf/src/google/protobuf/util/json_util.h> + #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/base/tabletid.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/util/yverify_stream.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/log.h> - + #include <util/digest/city.h> -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/ptr.h> -#include <util/generic/set.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/generic/set.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> #include <util/generic/algorithm.h> - -namespace NKikimr { -namespace NSchemeBoard { - -#define SBP_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_POPULATOR, stream) -#define SBP_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_POPULATOR, stream) -#define SBP_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_POPULATOR, stream) -#define SBP_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_POPULATOR, stream) + +namespace NKikimr { +namespace NSchemeBoard { + +#define SBP_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_POPULATOR, stream) +#define SBP_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_POPULATOR, stream) +#define SBP_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_POPULATOR, stream) +#define SBP_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_POPULATOR, stream) #define SBP_LOG_CRIT(stream) SB_LOG_CRIT(SCHEME_BOARD_POPULATOR, stream) - -namespace { - - using TDelayedUpdates = TVector<THolder<IEventHandle>>; - - void ReplayUpdates(TDelayedUpdates& updates) { - for (auto& update : updates) { - TlsActivationContext->Send(update.Release()); - } - - TDelayedUpdates().swap(updates); - } - -} // anonymous - -class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { - void ProcessSync(TSchemeBoardEvents::TEvDescribeResult* msg = nullptr, const TPathId& pathId = TPathId()) { - if (msg == nullptr) { - BatchSize = 0; - Send(Parent, new TSchemeBoardEvents::TEvRequestDescribe(pathId, Replica)); - return; - } - - if (msg->Commit) { - auto commit = MakeHolder<TSchemeBoardEvents::TEvCommitRequest>(Owner, Generation); - Send(Replica, std::move(commit), IEventHandle::FlagTrackDelivery); - return; - } - - auto update = msg->HasDescription() - ? MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, msg->Description.Record) - : MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation); - - if (msg->HasDeletedLocalPathIds()) { - auto& deletedLocalPathIds = *update->Record.MutableDeletedLocalPathIds(); - + +namespace { + + using TDelayedUpdates = TVector<THolder<IEventHandle>>; + + void ReplayUpdates(TDelayedUpdates& updates) { + for (auto& update : updates) { + TlsActivationContext->Send(update.Release()); + } + + TDelayedUpdates().swap(updates); + } + +} // anonymous + +class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { + void ProcessSync(TSchemeBoardEvents::TEvDescribeResult* msg = nullptr, const TPathId& pathId = TPathId()) { + if (msg == nullptr) { + BatchSize = 0; + Send(Parent, new TSchemeBoardEvents::TEvRequestDescribe(pathId, Replica)); + return; + } + + if (msg->Commit) { + auto commit = MakeHolder<TSchemeBoardEvents::TEvCommitRequest>(Owner, Generation); + Send(Replica, std::move(commit), IEventHandle::FlagTrackDelivery); + return; + } + + auto update = msg->HasDescription() + ? MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, msg->Description.Record) + : MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation); + + if (msg->HasDeletedLocalPathIds()) { + auto& deletedLocalPathIds = *update->Record.MutableDeletedLocalPathIds(); + deletedLocalPathIds.SetBegin(msg->DeletedPathBegin); deletedLocalPathIds.SetEnd(msg->DeletedPathEnd); - + CurPathId = TPathId(Owner, msg->DeletedPathEnd); - } - - if (msg->HasDescription()) { - auto& record = msg->Description.Record; - - if (!record.HasStatus()) { - SBP_LOG_E("Ignore description without status" - << ": self# " << SelfId()); + } + + if (msg->HasDescription()) { + auto& record = msg->Description.Record; + + if (!record.HasStatus()) { + SBP_LOG_E("Ignore description without status" + << ": self# " << SelfId()); } else if (record.GetStatus() != NKikimrScheme::StatusSuccess) { - SBP_LOG_E("Ignore description" - << ": self# " << SelfId() + SBP_LOG_E("Ignore description" + << ": self# " << SelfId() << ", status# " << record.GetStatus() << ", msg# " << record.ShortDebugString()); - } else { + } else { CurPathId = GetPathId(record); - update->SetDescribeSchemeResult(std::move(msg->Description)); - } - } - + update->SetDescribeSchemeResult(std::move(msg->Description)); + } + } + if (msg->HasMigratedPath()) { SBP_LOG_D("Ignore description of migrated path" - << ": self# " << SelfId() - << ", owner# " << Owner - << ", localPathId# " << msg->MigratedPathId); + << ": self# " << SelfId() + << ", owner# " << Owner + << ", localPathId# " << msg->MigratedPathId); //this path should be described by another owner (tenant schemeshard) auto& migratedLocalPathIds = *update->Record.MutableMigratedLocalPathIds(); migratedLocalPathIds.SetBegin(msg->MigratedPathId); @@ -106,457 +106,457 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { CurPathId = TPathId(Owner, msg->MigratedPathId); } - if (++BatchSize < BatchSizeLimit) { + if (++BatchSize < BatchSizeLimit) { CurPathId = CurPathId.NextId(); Send(Parent, new TSchemeBoardEvents::TEvRequestDescribe(CurPathId, Replica)); - } else { - update->Record.SetNeedAck(true); - BatchSize = 0; - } - - Send(Replica, std::move(update), IEventHandle::FlagTrackDelivery); - } - - void ResumeSync(const TPathId& fromPathId) { - ProcessSync(nullptr, fromPathId); - } - - void EnqueueUpdate(TSchemeBoardEvents::TEvUpdate::TPtr& ev, bool canSend = false) { - const TPathId pathId = ev->Get()->GetPathId(); - const auto& record = (static_cast<TSchemeBoardEvents::TEvUpdateBuilder*>(ev->Get()))->Record; - const ui64 version = record.GetIsDeletion() ? Max<ui64>() : GetPathVersion(record.GetDescribeSchemeResult()); - - if (canSend && UpdatesInFlight.size() < BatchSizeLimit) { - bool needSend = true; - - auto it = UpdatesInFlight.find(pathId); - if (it != UpdatesInFlight.end() && !it->second.empty() && it->second.rbegin()->first >= version) { - needSend = false; - } - - it = UpdatesRequested.find(pathId); - if (it != UpdatesRequested.end()) { - UpdatesInFlight[pathId].insert(it->second.begin(), it->second.end()); - UpdatesRequested.erase(it); - } - - UpdatesInFlight[pathId].emplace(version, ev->Cookie); - if (needSend) { - Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); - } - } else { - auto it = UpdatesRequested.find(pathId); - if (it != UpdatesRequested.end()) { - Updates[pathId].insert(it->second.begin(), it->second.end()); - UpdatesRequested.erase(it); - } - - Updates[pathId].emplace(version, ev->Cookie); - } - } - - void DequeueUpdate(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { - const TPathId pathId = ev->Get()->GetPathId(); - const ui64 version = ev->Get()->Record.GetVersion(); - - auto it = UpdatesInFlight.find(pathId); - if (it != UpdatesInFlight.end()) { - auto& versions = it->second; - TSet<ui64> txIds; - - for (auto x = versions.begin(), end = versions.upper_bound(std::make_pair(version, Max<ui64>())); x != end;) { - txIds.insert(x->second); - versions.erase(x++); - } - - if (versions.empty()) { - UpdatesInFlight.erase(it); - } - - for (ui64 txId : txIds) { - Send(Parent, new TSchemeBoardEvents::TEvUpdateAck(Owner, Generation, pathId, version), 0, txId); - } - } - - if (UpdatesInFlight.size() < BatchSizeLimit) { - RequestUpdate(); - } - } - - bool RequestUpdate() { - if (UpdatesRequested.size() >= BatchSizeLimit) { - return false; - } - - auto it = Updates.begin(); - if (it == Updates.end()) { - return false; - } - - UpdatesRequested[it->first].insert(it->second.begin(), it->second.end()); - Send(Parent, new TSchemeBoardEvents::TEvRequestUpdate(it->first)); - Updates.erase(it); - - return true; - } - - void RequestUpdates() { - auto move = [](auto& src, auto& dst) { - dst.insert(src.begin(), src.end()); - src.clear(); - }; - - move(UpdatesRequested, Updates); - move(UpdatesInFlight, Updates); - while (RequestUpdate()); - } - - template <typename TEvent, typename T> - bool Check(TEvent& ev, T this_, T that, const TString& what) { - if (this_ != that) { - SBP_LOG_E("Suspicious " << TypeName<TEvent>() - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", " << what << "# " << this_ - << ", other " << what << "# " << that); - return false; - } - - return true; - } - - template <typename TEvent> - bool CheckOwner(TEvent& ev) { - return Check(ev, Owner, ev->Get()->Record.GetOwner(), "owner"); - } - - template <typename TEvent> - bool CheckGeneration(TEvent& ev) { - return Check(ev, Generation, ev->Get()->Record.GetGeneration(), "generation"); - } - - void Handle(TSchemeBoardEvents::TEvHandshakeResponse::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvHandshakeResponse" - << ": self# " << SelfId() - << ", sender# " << ev->Sender); - - if (!CheckOwner(ev)) { - return; - } - - const ui64 generation = ev->Get()->Record.GetGeneration(); - if (generation > Generation) { - SBP_LOG_CRIT("I MUST DIE" - << ": self# " << SelfId()); - Become(&TThis::StateCalm); - } else { - SBP_LOG_N("Successful handshake" - << ": self# " << SelfId() - << ", replica# " << ev->Sender); - - if (generation < Generation) { - SBP_LOG_N("Start full sync" - << ": self# " << SelfId() - << ", replica# " << ev->Sender); - ProcessSync(); - } else { - SBP_LOG_N("Resume sync" - << ": self# " << SelfId() - << ", replica# " << ev->Sender - << ", fromPathId# " << LastAckedPathId.NextId()); - ResumeSync(LastAckedPathId.NextId()); - } - - RequestUpdates(); - Become(&TThis::StateWork); - } - } - - void Handle(TSchemeBoardEvents::TEvDescribeResult::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvDescribeResult" - << ": self# " << SelfId() - << ", sender# " << ev->Sender); - SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - - ProcessSync(ev->Get()); - } - - void Handle(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdate" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - - EnqueueUpdate(ev, true); - } - - void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdateAck (replica)" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - - if (!CheckOwner(ev) || !CheckGeneration(ev)) { - return; - } - - if (!ev->Cookie && ev->Get()->GetPathId() == CurPathId) { - LastAckedPathId = CurPathId; + } else { + update->Record.SetNeedAck(true); + BatchSize = 0; + } + + Send(Replica, std::move(update), IEventHandle::FlagTrackDelivery); + } + + void ResumeSync(const TPathId& fromPathId) { + ProcessSync(nullptr, fromPathId); + } + + void EnqueueUpdate(TSchemeBoardEvents::TEvUpdate::TPtr& ev, bool canSend = false) { + const TPathId pathId = ev->Get()->GetPathId(); + const auto& record = (static_cast<TSchemeBoardEvents::TEvUpdateBuilder*>(ev->Get()))->Record; + const ui64 version = record.GetIsDeletion() ? Max<ui64>() : GetPathVersion(record.GetDescribeSchemeResult()); + + if (canSend && UpdatesInFlight.size() < BatchSizeLimit) { + bool needSend = true; + + auto it = UpdatesInFlight.find(pathId); + if (it != UpdatesInFlight.end() && !it->second.empty() && it->second.rbegin()->first >= version) { + needSend = false; + } + + it = UpdatesRequested.find(pathId); + if (it != UpdatesRequested.end()) { + UpdatesInFlight[pathId].insert(it->second.begin(), it->second.end()); + UpdatesRequested.erase(it); + } + + UpdatesInFlight[pathId].emplace(version, ev->Cookie); + if (needSend) { + Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); + } + } else { + auto it = UpdatesRequested.find(pathId); + if (it != UpdatesRequested.end()) { + Updates[pathId].insert(it->second.begin(), it->second.end()); + UpdatesRequested.erase(it); + } + + Updates[pathId].emplace(version, ev->Cookie); + } + } + + void DequeueUpdate(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { + const TPathId pathId = ev->Get()->GetPathId(); + const ui64 version = ev->Get()->Record.GetVersion(); + + auto it = UpdatesInFlight.find(pathId); + if (it != UpdatesInFlight.end()) { + auto& versions = it->second; + TSet<ui64> txIds; + + for (auto x = versions.begin(), end = versions.upper_bound(std::make_pair(version, Max<ui64>())); x != end;) { + txIds.insert(x->second); + versions.erase(x++); + } + + if (versions.empty()) { + UpdatesInFlight.erase(it); + } + + for (ui64 txId : txIds) { + Send(Parent, new TSchemeBoardEvents::TEvUpdateAck(Owner, Generation, pathId, version), 0, txId); + } + } + + if (UpdatesInFlight.size() < BatchSizeLimit) { + RequestUpdate(); + } + } + + bool RequestUpdate() { + if (UpdatesRequested.size() >= BatchSizeLimit) { + return false; + } + + auto it = Updates.begin(); + if (it == Updates.end()) { + return false; + } + + UpdatesRequested[it->first].insert(it->second.begin(), it->second.end()); + Send(Parent, new TSchemeBoardEvents::TEvRequestUpdate(it->first)); + Updates.erase(it); + + return true; + } + + void RequestUpdates() { + auto move = [](auto& src, auto& dst) { + dst.insert(src.begin(), src.end()); + src.clear(); + }; + + move(UpdatesRequested, Updates); + move(UpdatesInFlight, Updates); + while (RequestUpdate()); + } + + template <typename TEvent, typename T> + bool Check(TEvent& ev, T this_, T that, const TString& what) { + if (this_ != that) { + SBP_LOG_E("Suspicious " << TypeName<TEvent>() + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", " << what << "# " << this_ + << ", other " << what << "# " << that); + return false; + } + + return true; + } + + template <typename TEvent> + bool CheckOwner(TEvent& ev) { + return Check(ev, Owner, ev->Get()->Record.GetOwner(), "owner"); + } + + template <typename TEvent> + bool CheckGeneration(TEvent& ev) { + return Check(ev, Generation, ev->Get()->Record.GetGeneration(), "generation"); + } + + void Handle(TSchemeBoardEvents::TEvHandshakeResponse::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvHandshakeResponse" + << ": self# " << SelfId() + << ", sender# " << ev->Sender); + + if (!CheckOwner(ev)) { + return; + } + + const ui64 generation = ev->Get()->Record.GetGeneration(); + if (generation > Generation) { + SBP_LOG_CRIT("I MUST DIE" + << ": self# " << SelfId()); + Become(&TThis::StateCalm); + } else { + SBP_LOG_N("Successful handshake" + << ": self# " << SelfId() + << ", replica# " << ev->Sender); + + if (generation < Generation) { + SBP_LOG_N("Start full sync" + << ": self# " << SelfId() + << ", replica# " << ev->Sender); + ProcessSync(); + } else { + SBP_LOG_N("Resume sync" + << ": self# " << SelfId() + << ", replica# " << ev->Sender + << ", fromPathId# " << LastAckedPathId.NextId()); + ResumeSync(LastAckedPathId.NextId()); + } + + RequestUpdates(); + Become(&TThis::StateWork); + } + } + + void Handle(TSchemeBoardEvents::TEvDescribeResult::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvDescribeResult" + << ": self# " << SelfId() + << ", sender# " << ev->Sender); + SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); + + ProcessSync(ev->Get()); + } + + void Handle(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdate" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); + + EnqueueUpdate(ev, true); + } + + void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdateAck (replica)" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); + + if (!CheckOwner(ev) || !CheckGeneration(ev)) { + return; + } + + if (!ev->Cookie && ev->Get()->GetPathId() == CurPathId) { + LastAckedPathId = CurPathId; CurPathId = CurPathId.NextId(); Send(Parent, new TSchemeBoardEvents::TEvRequestDescribe(CurPathId, Replica)); - } - - DequeueUpdate(ev); - } - - void Handle(TSchemeBoardEvents::TEvCommitResponse::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvCommitResponse" - << ": self# " << SelfId() - << ", sender# " << ev->Sender); - SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - - if (!CheckOwner(ev) || !CheckGeneration(ev)) { - return; - } - - LastAckedPathId = CurPathId.PrevId(); - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); - - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); - auto& record = *response->Record.MutableReplicaPopulatorResponse(); - - ActorIdToProto(Parent, record.MutableParent()); - ActorIdToProto(Replica, record.MutableReplica()); - - record.SetOwner(Owner); - record.SetGeneration(Generation); - record.SetBatchSize(BatchSize); - record.SetBatchSizeLimit(BatchSizeLimit); - - record.MutableCurPathId()->SetOwnerId(CurPathId.OwnerId); - record.MutableCurPathId()->SetLocalPathId(CurPathId.LocalPathId); - - record.MutableLastAckedPathId()->SetOwnerId(LastAckedPathId.OwnerId); - record.MutableLastAckedPathId()->SetLocalPathId(LastAckedPathId.LocalPathId); - - auto fillUpdates = [limit, &response = response->Record](const auto& from, auto& to) { - ui32 count = 0; - for (const auto& [pathId, versions] : from) { - auto& update = *to.Add(); - - update.MutablePathId()->SetOwnerId(pathId.OwnerId); - update.MutablePathId()->SetLocalPathId(pathId.LocalPathId); - - NKikimrSchemeBoardMon::TReplicaPopulatorResponse::TUpdateInfo::TVersionInfo* info = nullptr; - TMaybe<ui64> prevVersion; - - for (const auto& [version, txId] : versions) { - if (!prevVersion || *prevVersion != version) { - info = update.AddVersions(); - } - - Y_VERIFY(info); - info->SetVersion(version); - info->AddTxIds(txId); - prevVersion = version; - } - - if (++count >= limit) { - response.SetTruncated(true); - break; - } - } - }; - - fillUpdates(Updates, *record.MutableUpdates()); - fillUpdates(UpdatesRequested, *record.MutableUpdatesRequested()); - fillUpdates(UpdatesInFlight, *record.MutableUpdatesInFlight()); - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void SomeSleep() { - Become(&TThis::StateSleep, TDuration::MilliSeconds(50), new TEvents::TEvWakeup()); - } - - void PassAway() override { - if (Replica.NodeId() != SelfId().NodeId()) { - Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); - } - - TMonitorableActor::PassAway(); - } - - NJson::TJsonMap MonAttributes() const override { - return { - {"Parent", PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_POPULATOR_ACTOR, Parent)}, - {"Replica", PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, - {"Owner", Owner}, - {"Generation", Generation}, - }; - } - -public: + } + + DequeueUpdate(ev); + } + + void Handle(TSchemeBoardEvents::TEvCommitResponse::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvCommitResponse" + << ": self# " << SelfId() + << ", sender# " << ev->Sender); + SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); + + if (!CheckOwner(ev) || !CheckGeneration(ev)) { + return; + } + + LastAckedPathId = CurPathId.PrevId(); + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); + + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); + auto& record = *response->Record.MutableReplicaPopulatorResponse(); + + ActorIdToProto(Parent, record.MutableParent()); + ActorIdToProto(Replica, record.MutableReplica()); + + record.SetOwner(Owner); + record.SetGeneration(Generation); + record.SetBatchSize(BatchSize); + record.SetBatchSizeLimit(BatchSizeLimit); + + record.MutableCurPathId()->SetOwnerId(CurPathId.OwnerId); + record.MutableCurPathId()->SetLocalPathId(CurPathId.LocalPathId); + + record.MutableLastAckedPathId()->SetOwnerId(LastAckedPathId.OwnerId); + record.MutableLastAckedPathId()->SetLocalPathId(LastAckedPathId.LocalPathId); + + auto fillUpdates = [limit, &response = response->Record](const auto& from, auto& to) { + ui32 count = 0; + for (const auto& [pathId, versions] : from) { + auto& update = *to.Add(); + + update.MutablePathId()->SetOwnerId(pathId.OwnerId); + update.MutablePathId()->SetLocalPathId(pathId.LocalPathId); + + NKikimrSchemeBoardMon::TReplicaPopulatorResponse::TUpdateInfo::TVersionInfo* info = nullptr; + TMaybe<ui64> prevVersion; + + for (const auto& [version, txId] : versions) { + if (!prevVersion || *prevVersion != version) { + info = update.AddVersions(); + } + + Y_VERIFY(info); + info->SetVersion(version); + info->AddTxIds(txId); + prevVersion = version; + } + + if (++count >= limit) { + response.SetTruncated(true); + break; + } + } + }; + + fillUpdates(Updates, *record.MutableUpdates()); + fillUpdates(UpdatesRequested, *record.MutableUpdatesRequested()); + fillUpdates(UpdatesInFlight, *record.MutableUpdatesInFlight()); + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void SomeSleep() { + Become(&TThis::StateSleep, TDuration::MilliSeconds(50), new TEvents::TEvWakeup()); + } + + void PassAway() override { + if (Replica.NodeId() != SelfId().NodeId()) { + Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); + } + + TMonitorableActor::PassAway(); + } + + NJson::TJsonMap MonAttributes() const override { + return { + {"Parent", PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_POPULATOR_ACTOR, Parent)}, + {"Replica", PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, + {"Owner", Owner}, + {"Generation", Generation}, + }; + } + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_POPULATOR_ACTOR; - } - - explicit TReplicaPopulator( + } + + explicit TReplicaPopulator( const TActorId& parent, const TActorId& replica, const ui64 owner, const ui64 generation) - : Parent(parent) - , Replica(replica) - , Owner(owner) - , Generation(generation) - , BatchSize(0) - { - } - - void Bootstrap() { - TMonitorableActor::Bootstrap(); - - auto handshake = MakeHolder<TSchemeBoardEvents::TEvHandshakeRequest>(Owner, Generation); - Send(Replica, std::move(handshake), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession); - Become(&TThis::StateHandshake); - } - - STATEFN(StateHandshake) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvHandshakeResponse, Handle); - hFunc(TSchemeBoardEvents::TEvUpdate, EnqueueUpdate); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, SomeSleep); - - cFunc(TEvents::TEvUndelivered::EventType, SomeSleep); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvDescribeResult, Handle); - hFunc(TSchemeBoardEvents::TEvUpdate, Handle); - hFunc(TSchemeBoardEvents::TEvUpdateAck, Handle); - hFunc(TSchemeBoardEvents::TEvCommitResponse, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, SomeSleep); - - cFunc(TEvents::TEvUndelivered::EventType, SomeSleep); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateSleep) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvUpdate, EnqueueUpdate); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvents::TEvWakeup::EventType, Bootstrap); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateCalm) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: + : Parent(parent) + , Replica(replica) + , Owner(owner) + , Generation(generation) + , BatchSize(0) + { + } + + void Bootstrap() { + TMonitorableActor::Bootstrap(); + + auto handshake = MakeHolder<TSchemeBoardEvents::TEvHandshakeRequest>(Owner, Generation); + Send(Replica, std::move(handshake), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession); + Become(&TThis::StateHandshake); + } + + STATEFN(StateHandshake) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvHandshakeResponse, Handle); + hFunc(TSchemeBoardEvents::TEvUpdate, EnqueueUpdate); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, SomeSleep); + + cFunc(TEvents::TEvUndelivered::EventType, SomeSleep); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvDescribeResult, Handle); + hFunc(TSchemeBoardEvents::TEvUpdate, Handle); + hFunc(TSchemeBoardEvents::TEvUpdateAck, Handle); + hFunc(TSchemeBoardEvents::TEvCommitResponse, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, SomeSleep); + + cFunc(TEvents::TEvUndelivered::EventType, SomeSleep); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateSleep) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvUpdate, EnqueueUpdate); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvents::TEvWakeup::EventType, Bootstrap); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateCalm) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: const TActorId Parent; const TActorId Replica; - const ui64 Owner; - const ui64 Generation; - - // TPathId: [version; txId] - THashMap<TPathId, TSet<std::pair<ui64, ui64>>> Updates; - THashMap<TPathId, TSet<std::pair<ui64, ui64>>> UpdatesRequested; - THashMap<TPathId, TSet<std::pair<ui64, ui64>>> UpdatesInFlight; - - // used during sync + const ui64 Owner; + const ui64 Generation; + + // TPathId: [version; txId] + THashMap<TPathId, TSet<std::pair<ui64, ui64>>> Updates; + THashMap<TPathId, TSet<std::pair<ui64, ui64>>> UpdatesRequested; + THashMap<TPathId, TSet<std::pair<ui64, ui64>>> UpdatesInFlight; + + // used during sync TPathId CurPathId; - TPathId LastAckedPathId; - ui32 BatchSize; - static constexpr ui32 BatchSizeLimit = 100; - -}; // TReplicaPopulator - -class TPopulator: public TMonitorableActor<TPopulator> { + TPathId LastAckedPathId; + ui32 BatchSize; + static constexpr ui32 BatchSizeLimit = 100; + +}; // TReplicaPopulator + +class TPopulator: public TMonitorableActor<TPopulator> { TConstArrayRef<TActorId> SelectReplicas(TPathId pathId, TStringBuf path) { SelectionReplicaCache.clear(); - + const ui64 pathHash = CityHash64(path); const ui64 idHash = pathId.Hash(); - + TStateStorageInfo::TSelection selection; - + GroupInfo->SelectReplicas(pathHash, &selection); SelectionReplicaCache.insert(SelectionReplicaCache.end(), selection.begin(), selection.end()); - + GroupInfo->SelectReplicas(idHash, &selection); for (const TActorId& replica : selection) { - if (Find(SelectionReplicaCache, replica) == SelectionReplicaCache.end()) { + if (Find(SelectionReplicaCache, replica) == SelectionReplicaCache.end()) { SelectionReplicaCache.emplace_back(replica); - } - } + } + } - if (SelectionReplicaCache) { + if (SelectionReplicaCache) { return TConstArrayRef<TActorId>(&SelectionReplicaCache.front(), SelectionReplicaCache.size()); - } else { + } else { return TConstArrayRef<TActorId>(); - } + } } void Update(const TPathId pathId, const bool isDeletion, const ui64 cookie) { - auto it = Descriptions.find(pathId); - Y_VERIFY(it != Descriptions.end()); + auto it = Descriptions.find(pathId); + Y_VERIFY(it != Descriptions.end()); + + const auto& record = it->second.Record; - const auto& record = it->second.Record; - TConstArrayRef<TActorId> replicas = SelectReplicas(pathId, record.GetPath()); for (const auto& replica : replicas) { const TActorId* replicaPopulator = ReplicaToReplicaPopulator.FindPtr(replica); Y_VERIFY(replicaPopulator != nullptr); - - auto update = MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, record, isDeletion); - if (!isDeletion) { - update->SetDescribeSchemeResult(it->second); - } - update->Record.SetNeedAck(true); - - Send(*replicaPopulator, std::move(update), 0, cookie); - } - } - - void Handle(TSchemeBoardEvents::TEvRequestDescribe::TPtr& ev) { - SBP_LOG_D("Handle TSchemeBoardEvents::TEvRequestDescribe" - << ": self# " << SelfId() - << ", sender# " << ev->Sender + + auto update = MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, record, isDeletion); + if (!isDeletion) { + update->SetDescribeSchemeResult(it->second); + } + update->Record.SetNeedAck(true); + + Send(*replicaPopulator, std::move(update), 0, cookie); + } + } + + void Handle(TSchemeBoardEvents::TEvRequestDescribe::TPtr& ev) { + SBP_LOG_D("Handle TSchemeBoardEvents::TEvRequestDescribe" + << ": self# " << SelfId() + << ", sender# " << ev->Sender << ", pathId# " << ev->Get()->PathId); - + const TActorId replicaPopulator = ev->Sender; const TActorId replica = ev->Get()->Replica; - + if (ReplicaToReplicaPopulator[replica] != replicaPopulator) { - SBP_LOG_CRIT("Inconsistent replica populator" - << ": self# " << SelfId() - << ", replica# " << replica - << ", replicaPopulator# " << replicaPopulator); + SBP_LOG_CRIT("Inconsistent replica populator" + << ": self# " << SelfId() + << ", replica# " << replica + << ", replicaPopulator# " << replicaPopulator); return; } @@ -564,19 +564,19 @@ class TPopulator: public TMonitorableActor<TPopulator> { Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(true)); return; } - + TPathId startPathId = ev->Get()->PathId; - auto it = Descriptions.end(); + auto it = Descriptions.end(); if (ev->Get()->PathId) { startPathId = ev->Get()->PathId; - it = Descriptions.lower_bound(startPathId); + it = Descriptions.lower_bound(startPathId); } else { - it = Descriptions.begin(); - startPathId = it->first; + it = Descriptions.begin(); + startPathId = it->first; } - while (it != Descriptions.end()) { // skip irrelevant to the replica + while (it != Descriptions.end()) { // skip irrelevant to the replica if (it->second.Record.GetStatus() == NKikimrScheme::StatusPathDoesNotExist) { // KIKIMR-13173 // it is assumed that not deleted pathes present in Descriptions @@ -592,15 +592,15 @@ class TPopulator: public TMonitorableActor<TPopulator> { TConstArrayRef<TActorId> replicas = SelectReplicas(pathId, it->second.Record.GetPath()); if (Find(replicas, replica) != replicas.end()) { break; - } - ++it; - } - - if (it == Descriptions.end()) { + } + ++it; + } + + if (it == Descriptions.end()) { if (startPathId >= MaxPathId) { Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(true)); return; - } + } if (startPathId.OwnerId == Owner) { Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(startPathId.LocalPathId, MaxPathId.LocalPathId)); @@ -608,20 +608,20 @@ class TPopulator: public TMonitorableActor<TPopulator> { Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(1, MaxPathId.LocalPathId)); } return; - } - + } + const auto& description = it->second; const auto& record = description.Record; auto pathId = TPathId(it->second.Record.GetPathOwnerId(), it->second.Record.GetPathId()); if (pathId.OwnerId != Owner) { // this is an alien migrated migrated path from another owner, push it as a dot - Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(0, 0, description)); + Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(0, 0, description)); return; } TLocalPathId deletedBegin = startPathId.LocalPathId; - TLocalPathId deletedEnd = record.GetPathId() - 1; + TLocalPathId deletedEnd = record.GetPathId() - 1; if (startPathId.OwnerId != Owner) { deletedBegin = 1; @@ -635,370 +635,370 @@ class TPopulator: public TMonitorableActor<TPopulator> { if (record.GetStatus() == NKikimrScheme::EStatus::StatusRedirectDomain) { // this path has been migrated to another owner - Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(deletedBegin, deletedEnd, it->first.LocalPathId)); + Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(deletedBegin, deletedEnd, it->first.LocalPathId)); return; } - Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(deletedBegin, deletedEnd, description)); - } - - void Handle(TSchemeBoardEvents::TEvRequestUpdate::TPtr& ev) { - const TPathId pathId = ev->Get()->PathId; - - SBP_LOG_D("Handle TSchemeBoardEvents::TEvRequestUpdate" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", pathId# " << pathId); - - THolder<TSchemeBoardEvents::TEvUpdateBuilder> update; - - auto it = Descriptions.find(pathId); - if (it == Descriptions.end()) { + Send(replicaPopulator, new TSchemeBoardEvents::TEvDescribeResult(deletedBegin, deletedEnd, description)); + } + + void Handle(TSchemeBoardEvents::TEvRequestUpdate::TPtr& ev) { + const TPathId pathId = ev->Get()->PathId; + + SBP_LOG_D("Handle TSchemeBoardEvents::TEvRequestUpdate" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", pathId# " << pathId); + + THolder<TSchemeBoardEvents::TEvUpdateBuilder> update; + + auto it = Descriptions.find(pathId); + if (it == Descriptions.end()) { update = MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, pathId); - } else { + } else { update = MakeHolder<TSchemeBoardEvents::TEvUpdateBuilder>(Owner, Generation, it->second.Record); - update->SetDescribeSchemeResult(it->second); - } - - update->Record.SetNeedAck(true); - Send(ev->Sender, std::move(update)); - } - + update->SetDescribeSchemeResult(it->second); + } + + update->Record.SetNeedAck(true); + Send(ev->Sender, std::move(update)); + } + void DelayUpdate(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { SBP_LOG_D("DelayUpdate TEvSchemeShard::TEvDescribeSchemeResult" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - DelayedUpdates.emplace_back(ev.Release()); - } - + DelayedUpdates.emplace_back(ev.Release()); + } + void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { SBP_LOG_D("Handle TEvSchemeShard::TEvDescribeSchemeResult" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); - + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + SBP_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 1000)); + auto* msg = static_cast<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder*>(ev->Get()); - auto& record = msg->Record; - - if (!record.HasStatus()) { - SBP_LOG_E("Description without status" - << ": self# " << SelfId()); - return; - } - + auto& record = msg->Record; + + if (!record.HasStatus()) { + SBP_LOG_E("Description without status" + << ": self# " << SelfId()); + return; + } + const TPathId pathId = GetPathId(record); const bool isDeletion = record.GetStatus() == NKikimrScheme::StatusPathDoesNotExist; - const ui64 version = isDeletion ? Max<ui64>() : GetPathVersion(record); - - SBP_LOG_N("Update description" - << ": self# " << SelfId() - << ", owner# " << Owner + const ui64 version = isDeletion ? Max<ui64>() : GetPathVersion(record); + + SBP_LOG_N("Update description" + << ": self# " << SelfId() + << ", owner# " << Owner << ", pathId# " << pathId << ", cookie# " << ev->Cookie - << ", is deletion# " << (isDeletion ? "true" : "false")); - - if (isDeletion) { + << ", is deletion# " << (isDeletion ? "true" : "false")); + + if (isDeletion) { if (!Descriptions.contains(pathId)) { - SBP_LOG_N("Immediate ack for deleted path" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie + SBP_LOG_N("Immediate ack for deleted path" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie << ", pathId# " << pathId); - + auto ack = MakeHolder<TSchemeBoardEvents::TEvUpdateAck>(Owner, Generation, pathId, Max<ui64>()); - Send(ev->Sender, std::move(ack), 0, ev->Cookie); - return; - } - } else { - Descriptions[pathId] = TTwoPartDescription(std::move(msg->PreSerializedData), std::move(record)); + Send(ev->Sender, std::move(ack), 0, ev->Cookie); + return; + } + } else { + Descriptions[pathId] = TTwoPartDescription(std::move(msg->PreSerializedData), std::move(record)); MaxPathId = Max(MaxPathId, pathId.NextId()); - } - - auto it = UpdateAcks.find(ev->Cookie); - if (it == UpdateAcks.end()) { - it = UpdateAcks.emplace(ev->Cookie, TUpdateAckInfo{ev->Sender, {}}).first; - } - - it->second.AckTo = ev->Sender; + } + + auto it = UpdateAcks.find(ev->Cookie); + if (it == UpdateAcks.end()) { + it = UpdateAcks.emplace(ev->Cookie, TUpdateAckInfo{ev->Sender, {}}).first; + } + + it->second.AckTo = ev->Sender; it->second.PathAcks.emplace(std::make_pair(pathId, version), 0); - + Update(pathId, isDeletion, ev->Cookie); - - if (isDeletion) { + + if (isDeletion) { Descriptions.erase(pathId); - } - } - - void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { - const auto& record = ev->Get()->Record; - - SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdateAck (main)" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - SBP_LOG_T("Message:\n" << record.ShortDebugString()); - - auto it = UpdateAcks.find(ev->Cookie); - if (it == UpdateAcks.end()) { - SBP_LOG_D("Ack for unknown update (already acked?)" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - return; - } - + } + } + + void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { + const auto& record = ev->Get()->Record; + + SBP_LOG_D("Handle TSchemeBoardEvents::TEvUpdateAck (main)" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + SBP_LOG_T("Message:\n" << record.ShortDebugString()); + + auto it = UpdateAcks.find(ev->Cookie); + if (it == UpdateAcks.end()) { + SBP_LOG_D("Ack for unknown update (already acked?)" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + return; + } + const TPathId pathId = ev->Get()->GetPathId(); - const ui64 version = record.GetVersion(); - + const ui64 version = record.GetVersion(); + auto pathIt = it->second.PathAcks.lower_bound({pathId, 0}); - while (pathIt != it->second.PathAcks.end() + while (pathIt != it->second.PathAcks.end() && pathIt->first.first == pathId - && pathIt->first.second <= version) { - if (++pathIt->second > (GroupInfo->NToSelect / 2)) { - SBP_LOG_N("Ack update" - << ": self# " << SelfId() - << ", ack to# " << it->second.AckTo - << ", cookie# " << ev->Cookie + && pathIt->first.second <= version) { + if (++pathIt->second > (GroupInfo->NToSelect / 2)) { + SBP_LOG_N("Ack update" + << ": self# " << SelfId() + << ", ack to# " << it->second.AckTo + << ", cookie# " << ev->Cookie << ", pathId# " << pathId - << ", version# " << pathIt->first.second); - + << ", version# " << pathIt->first.second); + auto ack = MakeHolder<TSchemeBoardEvents::TEvUpdateAck>(Owner, Generation, pathId, pathIt->first.second); - Send(it->second.AckTo, std::move(ack), 0, ev->Cookie); - - auto eraseIt = pathIt; - ++pathIt; - it->second.PathAcks.erase(eraseIt); - - if (it->second.PathAcks.empty()) { - UpdateAcks.erase(it); - break; - } - } else { - ++pathIt; - } - } - } - - void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { - SBP_LOG_D("Handle TEvStateStorage::TEvListSchemeBoardResult" - << ": self# " << SelfId() - << ", sender# " << ev->Sender); - + Send(it->second.AckTo, std::move(ack), 0, ev->Cookie); + + auto eraseIt = pathIt; + ++pathIt; + it->second.PathAcks.erase(eraseIt); + + if (it->second.PathAcks.empty()) { + UpdateAcks.erase(it); + break; + } + } else { + ++pathIt; + } + } + } + + void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { + SBP_LOG_D("Handle TEvStateStorage::TEvListSchemeBoardResult" + << ": self# " << SelfId() + << ", sender# " << ev->Sender); + const auto& info = ev->Get()->Info; - + if (!info) { - SBP_LOG_E("Publish on unconfigured SchemeBoard" - << ": self# " << SelfId() - << ", StateStorage group# " << StateStorageGroup); - Become(&TThis::StateCalm); - return; - } - + SBP_LOG_E("Publish on unconfigured SchemeBoard" + << ": self# " << SelfId() + << ", StateStorage group# " << StateStorageGroup); + Become(&TThis::StateCalm); + return; + } + GroupInfo = info; - for (auto& replica : info->SelectAllReplicas()) { - IActor* replicaPopulator = new TReplicaPopulator(SelfId(), replica, Owner, Generation); - ReplicaToReplicaPopulator.emplace(replica, Register(replicaPopulator, TMailboxType::ReadAsFilled)); - } - + for (auto& replica : info->SelectAllReplicas()) { + IActor* replicaPopulator = new TReplicaPopulator(SelfId(), replica, Owner, Generation); + ReplicaToReplicaPopulator.emplace(replica, Register(replicaPopulator, TMailboxType::ReadAsFilled)); + } + Become(&TThis::StateWork); - ReplayUpdates(DelayedUpdates); - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); - - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); - auto& record = *response->Record.MutablePopulatorResponse(); - - record.SetOwner(Owner); - record.SetGeneration(Generation); - record.SetDescriptionsCount(Descriptions.size()); - - record.MutableMaxPathId()->SetOwnerId(MaxPathId.OwnerId); - record.MutableMaxPathId()->SetLocalPathId(MaxPathId.LocalPathId); - - record.SetDelayedUpdatesCount(DelayedUpdates.size()); - - for (const auto& [_, replicaPopulator] : ReplicaToReplicaPopulator) { - ActorIdToProto(replicaPopulator, record.MutableReplicaPopulators()->Add()); - } - - for (const auto& [cookie, info] : UpdateAcks) { - auto& updateAck = *record.AddUpdateAcks(); - - updateAck.SetCookie(cookie); - ActorIdToProto(info.AckTo, updateAck.MutableAckTo()); - - for (const auto& [pathIdVersion, acksCount] : info.PathAcks) { - auto& pathAck = *updateAck.AddPathAcks(); - - pathAck.MutablePathId()->SetOwnerId(pathIdVersion.first.OwnerId); - pathAck.MutablePathId()->SetLocalPathId(pathIdVersion.first.LocalPathId); - - pathAck.SetVersion(pathIdVersion.second); - pathAck.SetAcksCount(acksCount); - } - - if (record.UpdateAcksSize() >= limit) { - response->SetTruncated(); - break; - } - } - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - TTwoPartDescription* desc = nullptr; - if (record.HasPathId()) { - desc = Descriptions.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); - } - - TString json; - if (desc) { + ReplayUpdates(DelayedUpdates); + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); + + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); + auto& record = *response->Record.MutablePopulatorResponse(); + + record.SetOwner(Owner); + record.SetGeneration(Generation); + record.SetDescriptionsCount(Descriptions.size()); + + record.MutableMaxPathId()->SetOwnerId(MaxPathId.OwnerId); + record.MutableMaxPathId()->SetLocalPathId(MaxPathId.LocalPathId); + + record.SetDelayedUpdatesCount(DelayedUpdates.size()); + + for (const auto& [_, replicaPopulator] : ReplicaToReplicaPopulator) { + ActorIdToProto(replicaPopulator, record.MutableReplicaPopulators()->Add()); + } + + for (const auto& [cookie, info] : UpdateAcks) { + auto& updateAck = *record.AddUpdateAcks(); + + updateAck.SetCookie(cookie); + ActorIdToProto(info.AckTo, updateAck.MutableAckTo()); + + for (const auto& [pathIdVersion, acksCount] : info.PathAcks) { + auto& pathAck = *updateAck.AddPathAcks(); + + pathAck.MutablePathId()->SetOwnerId(pathIdVersion.first.OwnerId); + pathAck.MutablePathId()->SetLocalPathId(pathIdVersion.first.LocalPathId); + + pathAck.SetVersion(pathIdVersion.second); + pathAck.SetAcksCount(acksCount); + } + + if (record.UpdateAcksSize() >= limit) { + response->SetTruncated(); + break; + } + } + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + TTwoPartDescription* desc = nullptr; + if (record.HasPathId()) { + desc = Descriptions.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); + } + + TString json; + if (desc) { NKikimrScheme::TEvDescribeSchemeResult fullProto; Y_PROTOBUF_SUPPRESS_NODISCARD fullProto.ParseFromString(desc->PreSerialized); - fullProto.MergeFrom(desc->Record); - - using namespace google::protobuf::util; - - JsonPrintOptions opts; - opts.preserve_proto_field_names = true; - MessageToJsonString(fullProto, &json, opts); - } else { - json = "{}"; - } - - Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(json), 0, ev->Cookie); - } - - void HandleUndelivered() { - SBP_LOG_E("Publish on unavailable SchemeBoard" - << ": self# " << SelfId() - << ", StateStorage group# " << StateStorageGroup); - Become(&TThis::StateCalm); - } - - void PassAway() override { - for (const auto& x : ReplicaToReplicaPopulator) { - Send(x.second, new TEvents::TEvPoisonPill()); - } - - TMonitorableActor::PassAway(); - } - - NJson::TJsonMap MonAttributes() const override { - return { - {"Owner", Owner}, - {"Generation", Generation}, - }; - } - -public: + fullProto.MergeFrom(desc->Record); + + using namespace google::protobuf::util; + + JsonPrintOptions opts; + opts.preserve_proto_field_names = true; + MessageToJsonString(fullProto, &json, opts); + } else { + json = "{}"; + } + + Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(json), 0, ev->Cookie); + } + + void HandleUndelivered() { + SBP_LOG_E("Publish on unavailable SchemeBoard" + << ": self# " << SelfId() + << ", StateStorage group# " << StateStorageGroup); + Become(&TThis::StateCalm); + } + + void PassAway() override { + for (const auto& x : ReplicaToReplicaPopulator) { + Send(x.second, new TEvents::TEvPoisonPill()); + } + + TMonitorableActor::PassAway(); + } + + NJson::TJsonMap MonAttributes() const override { + return { + {"Owner", Owner}, + {"Generation", Generation}, + }; + } + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SCHEME_BOARD_POPULATOR_ACTOR; - } - - explicit TPopulator( - const ui64 owner, - const ui64 generation, + } + + explicit TPopulator( + const ui64 owner, + const ui64 generation, const ui32 ssId, - TMap<TPathId, TTwoPartDescription> descriptions, - const ui64 maxPathId) - : Owner(owner) - , Generation(generation) + TMap<TPathId, TTwoPartDescription> descriptions, + const ui64 maxPathId) + : Owner(owner) + , Generation(generation) , StateStorageGroup(ssId) - , Descriptions(std::move(descriptions)) + , Descriptions(std::move(descriptions)) , MaxPathId(TPathId(owner, maxPathId)) - { - } - - void Bootstrap() { - TMonitorableActor::Bootstrap(); - + { + } + + void Bootstrap() { + TMonitorableActor::Bootstrap(); + const TActorId proxy = MakeStateStorageProxyID(StateStorageGroup); - Send(proxy, new TEvStateStorage::TEvListSchemeBoard(), IEventHandle::FlagTrackDelivery); - Become(&TThis::StateResolve); - } - - STATEFN(StateResolve) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvStateStorage::TEvListSchemeBoardResult, Handle); - + Send(proxy, new TEvStateStorage::TEvListSchemeBoard(), IEventHandle::FlagTrackDelivery); + Become(&TThis::StateResolve); + } + + STATEFN(StateResolve) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvStateStorage::TEvListSchemeBoardResult, Handle); + hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, DelayUpdate); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); - - cFunc(TEvents::TEvUndelivered::EventType, HandleUndelivered); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvRequestDescribe, Handle); - hFunc(TSchemeBoardEvents::TEvRequestUpdate, Handle); - + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); + + cFunc(TEvents::TEvUndelivered::EventType, HandleUndelivered); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvRequestDescribe, Handle); + hFunc(TSchemeBoardEvents::TEvRequestUpdate, Handle); + hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); - hFunc(TSchemeBoardEvents::TEvUpdateAck, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateCalm) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - const ui64 Owner; - const ui64 Generation; - const ui64 StateStorageGroup; - - TMap<TPathId, TTwoPartDescription> Descriptions; + hFunc(TSchemeBoardEvents::TEvUpdateAck, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateCalm) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const ui64 Owner; + const ui64 Generation; + const ui64 StateStorageGroup; + + TMap<TPathId, TTwoPartDescription> Descriptions; TPathId MaxPathId; - - TDelayedUpdates DelayedUpdates; - + + TDelayedUpdates DelayedUpdates; + TIntrusiveConstPtr<TStateStorageInfo> GroupInfo; THashMap<TActorId, TActorId> ReplicaToReplicaPopulator; - + TVector<TActorId> SelectionReplicaCache; - - struct TUpdateAckInfo { + + struct TUpdateAckInfo { TActorId AckTo; TMap<std::pair<TPathId, ui64>, ui32> PathAcks; - }; - + }; + THashMap<ui64, TUpdateAckInfo> UpdateAcks; // ui64 is a cookie - -}; // TPopulator - -} // NSchemeBoard - -IActor* CreateSchemeBoardPopulator( - const ui64 owner, - const ui64 generation, + +}; // TPopulator + +} // NSchemeBoard + +IActor* CreateSchemeBoardPopulator( + const ui64 owner, + const ui64 generation, const ui32 ssId, - TMap<TPathId, NSchemeBoard::TTwoPartDescription> descriptions, - const ui64 maxPathId -) { + TMap<TPathId, NSchemeBoard::TTwoPartDescription> descriptions, + const ui64 maxPathId +) { return new NSchemeBoard::TPopulator(owner, generation, ssId, std::move(descriptions), maxPathId); -} - -} // NKikimr +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/populator.h b/ydb/core/tx/scheme_board/populator.h index f6f389549be..8105c56ba4b 100644 --- a/ydb/core/tx/scheme_board/populator.h +++ b/ydb/core/tx/scheme_board/populator.h @@ -1,20 +1,20 @@ -#pragma once - -#include "defs.h" -#include "two_part_description.h" - +#pragma once + +#include "defs.h" +#include "two_part_description.h" + #include <ydb/core/base/pathid.h> - -#include <util/generic/map.h> - -namespace NKikimr { - -IActor* CreateSchemeBoardPopulator( - const ui64 owner, - const ui64 generation, + +#include <util/generic/map.h> + +namespace NKikimr { + +IActor* CreateSchemeBoardPopulator( + const ui64 owner, + const ui64 generation, const ui32 schemeBoardSSId, - TMap<TPathId, NSchemeBoard::TTwoPartDescription> descriptions, - const ui64 maxPathId -); - -} // NKikimr + TMap<TPathId, NSchemeBoard::TTwoPartDescription> descriptions, + const ui64 maxPathId +); + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/populator_ut.cpp b/ydb/core/tx/scheme_board/populator_ut.cpp index 590f35b5f22..c0eb1cb62fa 100644 --- a/ydb/core/tx/scheme_board/populator_ut.cpp +++ b/ydb/core/tx/scheme_board/populator_ut.cpp @@ -1,130 +1,130 @@ -#include "events.h" -#include "ut_helpers.h" - +#include "events.h" +#include "ut_helpers.h" + #include <ydb/core/base/pathid.h> #include <ydb/core/base/tablet_types.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - + #include <library/cpp/actors/core/log.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/map.h> - -namespace NKikimr { -namespace NSchemeBoard { - -using namespace NSchemeShardUT_Private; - -class TPopulatorTest: public TTestWithSchemeshard { -public: - void SetUp() override { - TTestWithSchemeshard::SetUp(); - Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NLog::PRI_DEBUG); - } - - UNIT_TEST_SUITE(TPopulatorTest); - UNIT_TEST(Boot); - UNIT_TEST(MakeDir); - UNIT_TEST(RemoveDir); - UNIT_TEST_SUITE_END(); - - void Boot() { + +#include <util/generic/map.h> + +namespace NKikimr { +namespace NSchemeBoard { + +using namespace NSchemeShardUT_Private; + +class TPopulatorTest: public TTestWithSchemeshard { +public: + void SetUp() override { + TTestWithSchemeshard::SetUp(); + Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NLog::PRI_DEBUG); + } + + UNIT_TEST_SUITE(TPopulatorTest); + UNIT_TEST(Boot); + UNIT_TEST(MakeDir); + UNIT_TEST(RemoveDir); + UNIT_TEST_SUITE_END(); + + void Boot() { const TActorId edge = Context->AllocateEdgeActor(); - + NKikimr::TPathId rootPathId(TTestTxConfig::SchemeShard, RootPathId); - - Context->CreateSubscriber(edge, rootPathId); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("/Root", ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL(rootPathId, ev->Get()->PathId); - } - - void MakeDir() { + + Context->CreateSubscriber(edge, rootPathId); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("/Root", ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL(rootPathId, ev->Get()->PathId); + } + + void MakeDir() { const TActorId edge = Context->AllocateEdgeActor(); - - TestMkDir(*Context, 100, "/Root", "DirA"); - auto describe = DescribePath(*Context, "/Root/DirA"); - + + TestMkDir(*Context, 100, "/Root", "DirA"); + auto describe = DescribePath(*Context, "/Root/DirA"); + NKikimr::TPathId pathId(TTestTxConfig::SchemeShard, describe.GetPathId()); - - Context->CreateSubscriber(edge, pathId); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(describe.GetPath(), ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL(pathId, ev->Get()->PathId); - } - - void RemoveDir() { + + Context->CreateSubscriber(edge, pathId); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(describe.GetPath(), ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL(pathId, ev->Get()->PathId); + } + + void RemoveDir() { const TActorId edge = Context->AllocateEdgeActor(); - - TestMkDir(*Context, 100, "/Root", "DirB"); - auto describe = DescribePath(*Context, "/Root/DirB"); - + + TestMkDir(*Context, 100, "/Root", "DirB"); + auto describe = DescribePath(*Context, "/Root/DirB"); + NKikimr::TPathId pathId(TTestTxConfig::SchemeShard, describe.GetPathId()); - - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, pathId); - TestRmDir(*Context, 101, "/Root", "DirB"); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(describe.GetPath(), ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL(pathId, ev->Get()->PathId); - } - -}; // TPopulatorTest - -class TPopulatorTestWithResets: public TTestWithSchemeshard { -public: - void SetUp() override { - TTestWithSchemeshard::SetUp(); - Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NLog::PRI_DEBUG); - } - - TTestContext::TEventObserver ObserverFunc() override { - return [this](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TSchemeBoardEvents::EvHandshakeRequest: - ReplicaPopulators.emplace(ev->Sender, false); - runtime.EnableScheduleForActor(ev->Sender, true); - break; - - case TSchemeBoardEvents::EvUpdateAck: - auto it = ReplicaPopulators.find(ev->Recipient); - if (DropFirstAcks && it != ReplicaPopulators.end() && !it->second) { - it->second = true; - runtime.Send(new IEventHandle(ev->Recipient, ev->Sender, new TEvInterconnect::TEvNodeDisconnected(ev->Sender.NodeId()))); - - return TTestContext::EEventAction::DROP; - } - break; - } - - return TTestContext::EEventAction::PROCESS; - }; - } - - UNIT_TEST_SUITE(TPopulatorTestWithResets); - UNIT_TEST(UpdateAck); - UNIT_TEST_SUITE_END(); - - void UpdateAck() { - DropFirstAcks = true; - TestMkDir(*Context, 100, "/Root", "DirC"); + + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, pathId); + TestRmDir(*Context, 101, "/Root", "DirB"); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(describe.GetPath(), ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL(pathId, ev->Get()->PathId); + } + +}; // TPopulatorTest + +class TPopulatorTestWithResets: public TTestWithSchemeshard { +public: + void SetUp() override { + TTestWithSchemeshard::SetUp(); + Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NLog::PRI_DEBUG); + } + + TTestContext::TEventObserver ObserverFunc() override { + return [this](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TSchemeBoardEvents::EvHandshakeRequest: + ReplicaPopulators.emplace(ev->Sender, false); + runtime.EnableScheduleForActor(ev->Sender, true); + break; + + case TSchemeBoardEvents::EvUpdateAck: + auto it = ReplicaPopulators.find(ev->Recipient); + if (DropFirstAcks && it != ReplicaPopulators.end() && !it->second) { + it->second = true; + runtime.Send(new IEventHandle(ev->Recipient, ev->Sender, new TEvInterconnect::TEvNodeDisconnected(ev->Sender.NodeId()))); + + return TTestContext::EEventAction::DROP; + } + break; + } + + return TTestContext::EEventAction::PROCESS; + }; + } + + UNIT_TEST_SUITE(TPopulatorTestWithResets); + UNIT_TEST(UpdateAck); + UNIT_TEST_SUITE_END(); + + void UpdateAck() { + DropFirstAcks = true; + TestMkDir(*Context, 100, "/Root", "DirC"); TestWaitNotification(*Context, {100}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard)); - } - -private: + } + +private: TMap<TActorId, bool> ReplicaPopulators; - bool DropFirstAcks = false; - -}; // TPopulatorTestWithResets - -UNIT_TEST_SUITE_REGISTRATION(TPopulatorTest); -UNIT_TEST_SUITE_REGISTRATION(TPopulatorTestWithResets); - -} // NSchemeBoard -} // NKikimr + bool DropFirstAcks = false; + +}; // TPopulatorTestWithResets + +UNIT_TEST_SUITE_REGISTRATION(TPopulatorTest); +UNIT_TEST_SUITE_REGISTRATION(TPopulatorTestWithResets); + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/replica.cpp b/ydb/core/tx/scheme_board/replica.cpp index f51d765a9cc..638656e91db 100644 --- a/ydb/core/tx/scheme_board/replica.cpp +++ b/ydb/core/tx/scheme_board/replica.cpp @@ -1,171 +1,171 @@ -#include "double_indexed.h" -#include "events.h" -#include "helpers.h" -#include "monitorable_actor.h" -#include "replica.h" - -#include <contrib/libs/protobuf/src/google/protobuf/util/json_util.h> - +#include "double_indexed.h" +#include "events.h" +#include "helpers.h" +#include "monitorable_actor.h" +#include "replica.h" + +#include <contrib/libs/protobuf/src/google/protobuf/util/json_util.h> + #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/util/yverify_stream.h> - + #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/core/memory_track.h> - -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/variant.h> -#include <util/string/builder.h> - -namespace NKikimr { -namespace NSchemeBoard { - -#define SBR_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_REPLICA, stream) -#define SBR_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_REPLICA, stream) -#define SBR_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_REPLICA, stream) -#define SBR_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_REPLICA, stream) -#define SBR_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_REPLICA, stream) - -class TReplica: public TMonitorableActor<TReplica> { + +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/variant.h> +#include <util/string/builder.h> + +namespace NKikimr { +namespace NSchemeBoard { + +#define SBR_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_REPLICA, stream) +#define SBR_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_REPLICA, stream) +#define SBR_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_REPLICA, stream) +#define SBR_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_REPLICA, stream) +#define SBR_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_REPLICA, stream) + +class TReplica: public TMonitorableActor<TReplica> { using TDescribeSchemeResult = NKikimrScheme::TEvDescribeSchemeResult; - using TCapabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities; - -public: - enum ESubscriptionType { - SUBSCRIPTION_UNSPECIFIED, // for filtration - SUBSCRIPTION_BY_PATH, - SUBSCRIPTION_BY_PATH_ID, - }; - -private: - class TSubscriberInfo { - public: - explicit TSubscriberInfo(ESubscriptionType type, ui64 domainOwnerId, const TCapabilities& capabilities) - : Type(type) - , DomainOwnerId(domainOwnerId) - , Capabilities(capabilities) - , WaitForAck(false) - , LastVersionSent(0) - , NotifiedStrongly(true) + using TCapabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities; + +public: + enum ESubscriptionType { + SUBSCRIPTION_UNSPECIFIED, // for filtration + SUBSCRIPTION_BY_PATH, + SUBSCRIPTION_BY_PATH_ID, + }; + +private: + class TSubscriberInfo { + public: + explicit TSubscriberInfo(ESubscriptionType type, ui64 domainOwnerId, const TCapabilities& capabilities) + : Type(type) + , DomainOwnerId(domainOwnerId) + , Capabilities(capabilities) + , WaitForAck(false) + , LastVersionSent(0) + , NotifiedStrongly(true) , SyncRequestCookie(0) - , SyncResponseCookie(0) - { - } - - ESubscriptionType GetType() const { - return Type; - } - - ui64 GetDomainOwnerId() const { - return DomainOwnerId; - } - - const TCapabilities& GetCapabilities() const { - return Capabilities; - } - - bool EnqueueVersion(ui64 version, bool strong) { - if (!Capabilities.GetAckNotifications()) { - NotifiedStrongly = strong; - return true; - } - - if (WaitForAck) { - return false; - } - - WaitForAck = true; - LastVersionSent = version; - NotifiedStrongly = strong; - return true; - } - - bool EnqueueVersion(const TSchemeBoardEvents::TEvNotifyBuilder* notify) { - const auto& record = notify->Record; - return EnqueueVersion(record.GetVersion(), record.GetStrong()); - } - - bool AckVersion(ui64 version) { - if (LastVersionSent > version) { - return false; - } - - WaitForAck = false; - return true; - } - - bool IsWaitForAck() const { - return WaitForAck; - } - - void NeedStrongNotification() { - NotifiedStrongly = false; - } - - bool IsNotifiedStrongly() const { - return NotifiedStrongly; - } - - bool EnqueueSyncRequest(ui64 cookie) { - if (cookie <= SyncRequestCookie) { - return false; - } - - SyncRequestCookie = cookie; - return true; - } - - TMaybe<ui64> ProcessSyncRequest() { - if (SyncRequestCookie == SyncResponseCookie) { - return Nothing(); - } - - SyncResponseCookie = SyncRequestCookie; - return SyncResponseCookie; - } - - private: - const ESubscriptionType Type; - const ui64 DomainOwnerId; - const TCapabilities Capabilities; - - bool WaitForAck; - ui64 LastVersionSent; - bool NotifiedStrongly; - - ui64 SyncRequestCookie; - ui64 SyncResponseCookie; - }; - -public: - class TDescription { + , SyncResponseCookie(0) + { + } + + ESubscriptionType GetType() const { + return Type; + } + + ui64 GetDomainOwnerId() const { + return DomainOwnerId; + } + + const TCapabilities& GetCapabilities() const { + return Capabilities; + } + + bool EnqueueVersion(ui64 version, bool strong) { + if (!Capabilities.GetAckNotifications()) { + NotifiedStrongly = strong; + return true; + } + + if (WaitForAck) { + return false; + } + + WaitForAck = true; + LastVersionSent = version; + NotifiedStrongly = strong; + return true; + } + + bool EnqueueVersion(const TSchemeBoardEvents::TEvNotifyBuilder* notify) { + const auto& record = notify->Record; + return EnqueueVersion(record.GetVersion(), record.GetStrong()); + } + + bool AckVersion(ui64 version) { + if (LastVersionSent > version) { + return false; + } + + WaitForAck = false; + return true; + } + + bool IsWaitForAck() const { + return WaitForAck; + } + + void NeedStrongNotification() { + NotifiedStrongly = false; + } + + bool IsNotifiedStrongly() const { + return NotifiedStrongly; + } + + bool EnqueueSyncRequest(ui64 cookie) { + if (cookie <= SyncRequestCookie) { + return false; + } + + SyncRequestCookie = cookie; + return true; + } + + TMaybe<ui64> ProcessSyncRequest() { + if (SyncRequestCookie == SyncResponseCookie) { + return Nothing(); + } + + SyncResponseCookie = SyncRequestCookie; + return SyncResponseCookie; + } + + private: + const ESubscriptionType Type; + const ui64 DomainOwnerId; + const TCapabilities Capabilities; + + bool WaitForAck; + ui64 LastVersionSent; + bool NotifiedStrongly; + + ui64 SyncRequestCookie; + ui64 SyncResponseCookie; + }; + +public: + class TDescription { static constexpr char MemoryLabelDescribeResult[] = "SchemeBoard/Replica/DescribeSchemeResult"; - void Notify() { - if (!Subscribers) { - return; - } - - auto notify = BuildNotify(); + void Notify() { + if (!Subscribers) { + return; + } + + auto notify = BuildNotify(); TVector<const TActorId*> subscribers(Reserve(Subscribers.size())); - - for (auto& [subscriber, info] : Subscribers) { - if (!info.EnqueueVersion(notify.Get())) { - continue; - } - - subscribers.push_back(&subscriber); - } - - MultiSend(subscribers, Owner->SelfId(), std::move(notify)); - } - + + for (auto& [subscriber, info] : Subscribers) { + if (!info.EnqueueVersion(notify.Get())) { + continue; + } + + subscribers.push_back(&subscriber); + } + + MultiSend(subscribers, Owner->SelfId(), std::move(notify)); + } + void CalculateResultSize() { ResultSize = DescribeSchemeResult.ByteSizeLong(); } @@ -206,22 +206,22 @@ public: other.TrackMemory(); } - public: - explicit TDescription(TReplica* owner, const TString& path) - : Owner(owner) - , Path(path) - { + public: + explicit TDescription(TReplica* owner, const TString& path) + : Owner(owner) + , Path(path) + { TrackMemory(); - } - - explicit TDescription(TReplica* owner, const TPathId& pathId) - : Owner(owner) - , PathId(pathId) - { + } + + explicit TDescription(TReplica* owner, const TPathId& pathId) + : Owner(owner) + , PathId(pathId) + { TrackMemory(); - } - - explicit TDescription(TReplica* owner, const TString& path, const TPathId& pathId) + } + + explicit TDescription(TReplica* owner, const TString& path, const TPathId& pathId) : Owner(owner) , Path(path) , PathId(pathId) @@ -230,10 +230,10 @@ public: } explicit TDescription( - TReplica* owner, - const TPathId& pathId, - TDescribeSchemeResult&& describeSchemeResult) - : Owner(owner) + TReplica* owner, + const TPathId& pathId, + TDescribeSchemeResult&& describeSchemeResult) + : Owner(owner) , PathId(pathId) , DescribeSchemeResult(std::move(describeSchemeResult)) { @@ -242,19 +242,19 @@ public: } explicit TDescription( - TReplica* owner, + TReplica* owner, const TString& path, const TPathId& pathId, TDescribeSchemeResult&& describeSchemeResult) : Owner(owner) - , Path(path) - , PathId(pathId) - , DescribeSchemeResult(std::move(describeSchemeResult)) - { + , Path(path) + , PathId(pathId) + , DescribeSchemeResult(std::move(describeSchemeResult)) + { CalculateResultSize(); TrackMemory(); - } - + } + TDescription(TDescription&& other) { TrackMemory(); Move(std::move(other)); @@ -273,61 +273,61 @@ public: UntrackMemory(); } - bool operator<(const TDescription& other) const { - return GetVersion() < other.GetVersion(); - } - - bool operator>(const TDescription& other) const { - return other < *this; - } - - TDescription& Merge(TDescription&& other) noexcept { - Y_VERIFY(Owner == other.Owner); - - if (!Path) { - Path = other.Path; - } - - Y_VERIFY_S(!other.Path || Path == other.Path, "Descriptions" - << ": self# " << ToString() - << ", other# " << other.ToString()); - - if (!PathId) { - PathId = other.PathId; - } - - Y_VERIFY_S(!other.PathId || PathId == other.PathId, "Descriptions" - << ": self# " << ToString() - << ", other# " << other.ToString()); - + bool operator<(const TDescription& other) const { + return GetVersion() < other.GetVersion(); + } + + bool operator>(const TDescription& other) const { + return other < *this; + } + + TDescription& Merge(TDescription&& other) noexcept { + Y_VERIFY(Owner == other.Owner); + + if (!Path) { + Path = other.Path; + } + + Y_VERIFY_S(!other.Path || Path == other.Path, "Descriptions" + << ": self# " << ToString() + << ", other# " << other.ToString()); + + if (!PathId) { + PathId = other.PathId; + } + + Y_VERIFY_S(!other.PathId || PathId == other.PathId, "Descriptions" + << ": self# " << ToString() + << ", other# " << other.ToString()); + SBR_LOG_T("Merge descriptions" - << ": self# " << Owner->SelfId() - << ", left path# " << Path - << ", left pathId# " << PathId - << ", left version# " << GetVersion() - << ", rigth path# " << other.Path - << ", rigth pathId# " << other.PathId - << ", rigth version# " << other.GetVersion()); + << ": self# " << Owner->SelfId() + << ", left path# " << Path + << ", left pathId# " << PathId + << ", left version# " << GetVersion() + << ", rigth path# " << other.Path + << ", rigth pathId# " << other.PathId + << ", rigth version# " << other.GetVersion()); UntrackMemory(); other.UntrackMemory(); TrackNotify = false; other.TrackNotify = false; - if (*this > other) { - other.DescribeSchemeResult.Swap(&DescribeSchemeResult); - other.PreSerializedDescribeSchemeResult.Clear(); - other.ExplicitlyDeleted = ExplicitlyDeleted; - other.Notify(); - DescribeSchemeResult.Swap(&other.DescribeSchemeResult); - other.PreSerializedDescribeSchemeResult.Clear(); - } else if (*this < other) { - DescribeSchemeResult = std::move(other.DescribeSchemeResult); - PreSerializedDescribeSchemeResult.Clear(); - ExplicitlyDeleted = other.ExplicitlyDeleted; - Notify(); - } - + if (*this > other) { + other.DescribeSchemeResult.Swap(&DescribeSchemeResult); + other.PreSerializedDescribeSchemeResult.Clear(); + other.ExplicitlyDeleted = ExplicitlyDeleted; + other.Notify(); + DescribeSchemeResult.Swap(&other.DescribeSchemeResult); + other.PreSerializedDescribeSchemeResult.Clear(); + } else if (*this < other) { + DescribeSchemeResult = std::move(other.DescribeSchemeResult); + PreSerializedDescribeSchemeResult.Clear(); + ExplicitlyDeleted = other.ExplicitlyDeleted; + Notify(); + } + CalculateResultSize(); other.CalculateResultSize(); @@ -336,44 +336,44 @@ public: TrackMemory(); other.TrackMemory(); - Subscribers.insert(other.Subscribers.begin(), other.Subscribers.end()); - - return *this; - } - - const TDescribeSchemeResult& GetProto() const { - return DescribeSchemeResult; - } - - TString ToString() const { - return TStringBuilder() << "{" - << " Path# " << Path - << " PathId# " << PathId - << " DescribeSchemeResult# " << DescribeSchemeResult.ShortDebugString() - << " ExplicitlyDeleted# " << (ExplicitlyDeleted ? "true" : "false") - << " }"; - } - - const TString& GetPath() const { - return Path; - } - - const TPathId& GetPathId() const { - return PathId; - } - - bool IsExplicitlyDeleted() const { - return ExplicitlyDeleted; - } - - ui64 GetVersion() const { - if (ExplicitlyDeleted) { - return Max<ui64>(); - } - - return ::NKikimr::NSchemeBoard::GetPathVersion(DescribeSchemeResult); - } - + Subscribers.insert(other.Subscribers.begin(), other.Subscribers.end()); + + return *this; + } + + const TDescribeSchemeResult& GetProto() const { + return DescribeSchemeResult; + } + + TString ToString() const { + return TStringBuilder() << "{" + << " Path# " << Path + << " PathId# " << PathId + << " DescribeSchemeResult# " << DescribeSchemeResult.ShortDebugString() + << " ExplicitlyDeleted# " << (ExplicitlyDeleted ? "true" : "false") + << " }"; + } + + const TString& GetPath() const { + return Path; + } + + const TPathId& GetPathId() const { + return PathId; + } + + bool IsExplicitlyDeleted() const { + return ExplicitlyDeleted; + } + + ui64 GetVersion() const { + if (ExplicitlyDeleted) { + return Max<ui64>(); + } + + return ::NKikimr::NSchemeBoard::GetPathVersion(DescribeSchemeResult); + } + TDomainId GetDomainId() const { return IsFilled() ? ::NKikimr::NSchemeBoard::GetDomainId(DescribeSchemeResult) : TDomainId(); } @@ -382,198 +382,198 @@ public: return IsFilled() ? ::NKikimr::NSchemeBoard::GetAbandonedSchemeShardIds(DescribeSchemeResult) : TSet<ui64>(); } - bool IsFilled() const { - return DescribeSchemeResult.ByteSizeLong(); - } - - void Clear() { - ExplicitlyDeleted = true; + bool IsFilled() const { + return DescribeSchemeResult.ByteSizeLong(); + } + + void Clear() { + ExplicitlyDeleted = true; UntrackMemory(); - TDescribeSchemeResult().Swap(&DescribeSchemeResult); - PreSerializedDescribeSchemeResult.Clear(); + TDescribeSchemeResult().Swap(&DescribeSchemeResult); + PreSerializedDescribeSchemeResult.Clear(); ResultSize = 0; TrackMemory(); - Notify(); - } - - THolder<TSchemeBoardEvents::TEvNotifyBuilder> BuildNotify(bool forceStrong = false) const { - THolder<TSchemeBoardEvents::TEvNotifyBuilder> notify; - - const bool isDeletion = !IsFilled(); - - if (!PathId) { - Y_VERIFY(isDeletion); + Notify(); + } + + THolder<TSchemeBoardEvents::TEvNotifyBuilder> BuildNotify(bool forceStrong = false) const { + THolder<TSchemeBoardEvents::TEvNotifyBuilder> notify; + + const bool isDeletion = !IsFilled(); + + if (!PathId) { + Y_VERIFY(isDeletion); notify = MakeHolder<TSchemeBoardEvents::TEvNotifyBuilder>(Path, isDeletion); - } else if (!Path) { - Y_VERIFY(isDeletion); + } else if (!Path) { + Y_VERIFY(isDeletion); notify = MakeHolder<TSchemeBoardEvents::TEvNotifyBuilder>(PathId, isDeletion); - } else { + } else { notify = MakeHolder<TSchemeBoardEvents::TEvNotifyBuilder>(Path, PathId, isDeletion); - } - - if (!isDeletion) { - if (!PreSerializedDescribeSchemeResult) { - TString serialized; + } + + if (!isDeletion) { + if (!PreSerializedDescribeSchemeResult) { + TString serialized; Y_PROTOBUF_SUPPRESS_NODISCARD DescribeSchemeResult.SerializeToString(&serialized); if (TrackNotify) { UntrackMemory(); } - PreSerializedDescribeSchemeResult = std::move(serialized); + PreSerializedDescribeSchemeResult = std::move(serialized); if (TrackNotify) { TrackMemory(); } - } - - notify->SetDescribeSchemeResult(*PreSerializedDescribeSchemeResult); - } - - notify->Record.SetVersion(GetVersion()); - if (IsFilled() || IsExplicitlyDeleted() || forceStrong) { - notify->Record.SetStrong(true); - } - - return notify; - } - + } + + notify->SetDescribeSchemeResult(*PreSerializedDescribeSchemeResult); + } + + notify->Record.SetVersion(GetVersion()); + if (IsFilled() || IsExplicitlyDeleted() || forceStrong) { + notify->Record.SetStrong(true); + } + + return notify; + } + void Subscribe(const TActorId& subscriber, const TString&, ui64 domainOwnerId, const TCapabilities& capabilities) { - Subscribers.emplace(subscriber, TSubscriberInfo(SUBSCRIPTION_BY_PATH, domainOwnerId, capabilities)); - } - + Subscribers.emplace(subscriber, TSubscriberInfo(SUBSCRIPTION_BY_PATH, domainOwnerId, capabilities)); + } + void Subscribe(const TActorId& subscriber, const TPathId&, ui64 domainOwnerId, const TCapabilities& capabilities) { - Subscribers.emplace(subscriber, TSubscriberInfo(SUBSCRIPTION_BY_PATH_ID, domainOwnerId, capabilities)); - } - + Subscribers.emplace(subscriber, TSubscriberInfo(SUBSCRIPTION_BY_PATH_ID, domainOwnerId, capabilities)); + } + void Unsubscribe(const TActorId& subscriber) { - Subscribers.erase(subscriber); - } - + Subscribers.erase(subscriber); + } + TSubscriberInfo& GetSubscriberInfo(const TActorId& subscriber) { - auto it = Subscribers.find(subscriber); - Y_VERIFY(it != Subscribers.end()); - return it->second; - } - + auto it = Subscribers.find(subscriber); + Y_VERIFY(it != Subscribers.end()); + return it->second; + } + THashMap<TActorId, TSubscriberInfo> GetSubscribers(const ESubscriptionType type = SUBSCRIPTION_UNSPECIFIED) const { THashMap<TActorId, TSubscriberInfo> result; - - for (const auto& [subscriber, info] : Subscribers) { - if (type == SUBSCRIPTION_UNSPECIFIED || type == info.GetType()) { - result.emplace(subscriber, info); - } - } - - return result; - } - - private: - // used to notifications - TReplica* Owner; - - // data - TString Path; - TPathId PathId; - TDescribeSchemeResult DescribeSchemeResult; - mutable TMaybe<TString> PreSerializedDescribeSchemeResult; - bool ExplicitlyDeleted = false; - - // subscribers + + for (const auto& [subscriber, info] : Subscribers) { + if (type == SUBSCRIPTION_UNSPECIFIED || type == info.GetType()) { + result.emplace(subscriber, info); + } + } + + return result; + } + + private: + // used to notifications + TReplica* Owner; + + // data + TString Path; + TPathId PathId; + TDescribeSchemeResult DescribeSchemeResult; + mutable TMaybe<TString> PreSerializedDescribeSchemeResult; + bool ExplicitlyDeleted = false; + + // subscribers THashMap<TActorId, TSubscriberInfo> Subscribers; - + // memory tracking size_t ResultSize = 0; bool TrackNotify = true; - }; // TDescription - - struct TMerger { - TDescription& operator()(TDescription& dst, TDescription&& src) { - return dst.Merge(std::move(src)); - } - }; - -private: - struct TPopulatorInfo { + }; // TDescription + + struct TMerger { + TDescription& operator()(TDescription& dst, TDescription&& src) { + return dst.Merge(std::move(src)); + } + }; + +private: + struct TPopulatorInfo { ui64 Generation = 0; ui64 PendingGeneration = 0; TActorId PopulatorActor; - - bool IsCommited() const { - return Generation && Generation == PendingGeneration; - } - }; - - bool IsPopulatorCommited(ui64 ownerId) const { - auto it = Populators.find(ownerId); - if (it != Populators.end() && it->second.IsCommited()) { - return true; - } - - return false; - } - - template <typename TPath> - TDescription& UpsertDescription(const TPath& path) { - SBR_LOG_I("Upsert description" - << ": self# " << SelfId() - << ", path# " << path); - - return Descriptions.Upsert(path, TDescription(this, path)); - } - + + bool IsCommited() const { + return Generation && Generation == PendingGeneration; + } + }; + + bool IsPopulatorCommited(ui64 ownerId) const { + auto it = Populators.find(ownerId); + if (it != Populators.end() && it->second.IsCommited()) { + return true; + } + + return false; + } + + template <typename TPath> + TDescription& UpsertDescription(const TPath& path) { + SBR_LOG_I("Upsert description" + << ": self# " << SelfId() + << ", path# " << path); + + return Descriptions.Upsert(path, TDescription(this, path)); + } + TDescription& UpsertDescription(const TString& path, const TPathId& pathId) { SBR_LOG_I("Upsert description" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId); return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId)); } template <typename TPath> - TDescription& UpsertDescription(const TPath path, TDescription&& description) { + TDescription& UpsertDescription(const TPath path, TDescription&& description) { SBR_LOG_I("Upsert description" - << ": self# " << SelfId() - << ", path# " << path); + << ": self# " << SelfId() + << ", path# " << path); - return Descriptions.Upsert(path, std::move(description)); + return Descriptions.Upsert(path, std::move(description)); } TDescription& UpsertDescription( - const TString& path, - const TPathId& pathId, - TDescribeSchemeResult&& describeSchemeResult - ) { - SBR_LOG_I("Upsert description" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId); - - return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId, std::move(describeSchemeResult))); - } - - void SoftDeleteDescription(const TPathId& pathId, bool createIfNotExists = false) { - TDescription* desc = Descriptions.FindPtr(pathId); - - if (!desc) { - if (createIfNotExists) { - desc = &UpsertDescription(pathId); - desc->Clear(); // mark as deleted - } - - return; - } - - if (!desc->IsFilled()) { - return; - } - + const TString& path, + const TPathId& pathId, + TDescribeSchemeResult&& describeSchemeResult + ) { + SBR_LOG_I("Upsert description" + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId); + + return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId, std::move(describeSchemeResult))); + } + + void SoftDeleteDescription(const TPathId& pathId, bool createIfNotExists = false) { + TDescription* desc = Descriptions.FindPtr(pathId); + + if (!desc) { + if (createIfNotExists) { + desc = &UpsertDescription(pathId); + desc->Clear(); // mark as deleted + } + + return; + } + + if (!desc->IsFilled()) { + return; + } + auto path = desc->GetPath(); - SBR_LOG_I("Delete description" - << ": self# " << SelfId() + SBR_LOG_I("Delete description" + << ": self# " << SelfId() << ", path# " << path - << ", pathId# " << pathId); - + << ", pathId# " << pathId); + if (TDescription* descByPath = Descriptions.FindPtr(path)) { if (descByPath != desc && descByPath->IsFilled()) { if (descByPath->GetPathId().OwnerId != pathId.OwnerId) { @@ -583,272 +583,272 @@ private: if (curDomainId == pathId) { //Deletion from GSS SBR_LOG_N("Delete description by GSS" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); Descriptions.DeleteIndex(path); UpsertDescription(path, pathId); RelinkSubscribers(descByPath, path); - - descByPath->Clear(); + + descByPath->Clear(); } } } } - desc->Clear(); - } - + desc->Clear(); + } + void RelinkSubscribers(TDescription* fromDesc, const TString& path) { - for (const auto& [subscriber, info] : fromDesc->GetSubscribers(SUBSCRIPTION_BY_PATH)) { + for (const auto& [subscriber, info] : fromDesc->GetSubscribers(SUBSCRIPTION_BY_PATH)) { fromDesc->Unsubscribe(subscriber); - Subscribers.erase(subscriber); - SubscribeBy(subscriber, path, info.GetDomainOwnerId(), info.GetCapabilities(), false); + Subscribers.erase(subscriber); + SubscribeBy(subscriber, path, info.GetDomainOwnerId(), info.GetCapabilities(), false); + } + } + + void SoftDeleteDescriptions(const TPathId& begin, const TPathId& end) { + const auto& pathIdIndex = Descriptions.GetSecondaryIndex(); + + auto it = pathIdIndex.lower_bound(begin); + if (it == pathIdIndex.end()) { + return; + } + + const auto endIt = pathIdIndex.upper_bound(end); + while (it != endIt) { + SoftDeleteDescription(it->first); + ++it; } } - void SoftDeleteDescriptions(const TPathId& begin, const TPathId& end) { - const auto& pathIdIndex = Descriptions.GetSecondaryIndex(); - - auto it = pathIdIndex.lower_bound(begin); - if (it == pathIdIndex.end()) { - return; - } - - const auto endIt = pathIdIndex.upper_bound(end); - while (it != endIt) { - SoftDeleteDescription(it->first); - ++it; - } - } - - // call it _after_ Subscribe() & _before_ Unsubscribe() + // call it _after_ Subscribe() & _before_ Unsubscribe() bool IsSingleSubscriberOnNode(const TActorId& subscriber) const { - const ui32 nodeId = subscriber.NodeId(); + const ui32 nodeId = subscriber.NodeId(); auto it = Subscribers.lower_bound(TActorId(nodeId, 0, 0, 0)); - Y_VERIFY(it != Subscribers.end()); - - return ++it == Subscribers.end() || it->first.NodeId() != nodeId; - } - - template <typename TPath> + Y_VERIFY(it != Subscribers.end()); + + return ++it == Subscribers.end() || it->first.NodeId() != nodeId; + } + + template <typename TPath> void Subscribe(const TActorId& subscriber, const TPath& path, ui64 domainOwnerId, const TCapabilities& capabilities) { - TDescription* desc = Descriptions.FindPtr(path); - Y_VERIFY(desc); - - SBR_LOG_N("Subscribe" - << ": self# " << SelfId() - << ", subscriber# " << subscriber - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId - << ", capabilities# " << capabilities.ShortDebugString()); - - desc->Subscribe(subscriber, path, domainOwnerId, capabilities); - - auto it = Subscribers.find(subscriber); + TDescription* desc = Descriptions.FindPtr(path); + Y_VERIFY(desc); + + SBR_LOG_N("Subscribe" + << ": self# " << SelfId() + << ", subscriber# " << subscriber + << ", path# " << path + << ", domainOwnerId# " << domainOwnerId + << ", capabilities# " << capabilities.ShortDebugString()); + + desc->Subscribe(subscriber, path, domainOwnerId, capabilities); + + auto it = Subscribers.find(subscriber); Y_VERIFY_DEBUG(it == Subscribers.end() || std::holds_alternative<TPath>(it->second) && std::get<TPath>(it->second) == path); - Subscribers.emplace(subscriber, path); - } - - template <typename TPath> + Subscribers.emplace(subscriber, path); + } + + template <typename TPath> void Unsubscribe(const TActorId& subscriber, const TPath& path) { - TDescription* desc = Descriptions.FindPtr(path); - Y_VERIFY(desc); - - SBR_LOG_N("Unsubscribe" - << ": self# " << SelfId() - << ", subscriber# " << subscriber - << ", path# " << path); - - desc->Unsubscribe(subscriber); - Subscribers.erase(subscriber); - } - - template <typename TPath> + TDescription* desc = Descriptions.FindPtr(path); + Y_VERIFY(desc); + + SBR_LOG_N("Unsubscribe" + << ": self# " << SelfId() + << ", subscriber# " << subscriber + << ", path# " << path); + + desc->Unsubscribe(subscriber); + Subscribers.erase(subscriber); + } + + template <typename TPath> void SubscribeBy(const TActorId& subscriber, const TPath& path, ui64 domainOwnerId, const TCapabilities& capabilities, - bool needNotify = true) { - TDescription* desc = Descriptions.FindPtr(path); - if (!desc) { - desc = &UpsertDescription(path); - } - - Subscribe(subscriber, path, domainOwnerId, capabilities); - - if (!needNotify) { - return; - } - - ui32 flags = 0; - if (IsSingleSubscriberOnNode(subscriber)) { - flags = IEventHandle::FlagSubscribeOnSession; - } - - auto notify = desc->BuildNotify(IsPopulatorCommited(domainOwnerId)); - - if (!notify->Record.GetStrong()) { - auto& info = desc->GetSubscriberInfo(subscriber); - info.NeedStrongNotification(); - } - - Send(subscriber, std::move(notify), flags); - } - - template <typename TPath> + bool needNotify = true) { + TDescription* desc = Descriptions.FindPtr(path); + if (!desc) { + desc = &UpsertDescription(path); + } + + Subscribe(subscriber, path, domainOwnerId, capabilities); + + if (!needNotify) { + return; + } + + ui32 flags = 0; + if (IsSingleSubscriberOnNode(subscriber)) { + flags = IEventHandle::FlagSubscribeOnSession; + } + + auto notify = desc->BuildNotify(IsPopulatorCommited(domainOwnerId)); + + if (!notify->Record.GetStrong()) { + auto& info = desc->GetSubscriberInfo(subscriber); + info.NeedStrongNotification(); + } + + Send(subscriber, std::move(notify), flags); + } + + template <typename TPath> void UnsubscribeBy(const TActorId& subscriber, const TPath& path) { - if (!Descriptions.FindPtr(path) || !Subscribers.contains(subscriber)) { - return; - } - - if (IsSingleSubscriberOnNode(subscriber)) { - Send(MakeInterconnectProxyId(subscriber.NodeId()), new TEvents::TEvUnsubscribe()); - } - - Unsubscribe(subscriber, path); - } - - template <typename TPath> - ui64 GetVersion(const TPath& path) const { - const TDescription* desc = Descriptions.FindPtr(path); - return desc ? desc->GetVersion() : 0; - } - - void AckUpdate(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { - const auto& record = ev->Get()->GetRecord(); - - const ui64 owner = record.GetOwner(); - const ui64 generation = record.GetGeneration(); - - Y_VERIFY(Populators.contains(owner)); - Y_VERIFY(Populators.at(owner).PendingGeneration == generation); - - if (!record.GetNeedAck()) { - return; - } - + if (!Descriptions.FindPtr(path) || !Subscribers.contains(subscriber)) { + return; + } + + if (IsSingleSubscriberOnNode(subscriber)) { + Send(MakeInterconnectProxyId(subscriber.NodeId()), new TEvents::TEvUnsubscribe()); + } + + Unsubscribe(subscriber, path); + } + + template <typename TPath> + ui64 GetVersion(const TPath& path) const { + const TDescription* desc = Descriptions.FindPtr(path); + return desc ? desc->GetVersion() : 0; + } + + void AckUpdate(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { + const auto& record = ev->Get()->GetRecord(); + + const ui64 owner = record.GetOwner(); + const ui64 generation = record.GetGeneration(); + + Y_VERIFY(Populators.contains(owner)); + Y_VERIFY(Populators.at(owner).PendingGeneration == generation); + + if (!record.GetNeedAck()) { + return; + } + TPathId ackPathId; - - if (record.HasDeletedLocalPathIds()) { + + if (record.HasDeletedLocalPathIds()) { ackPathId = TPathId(owner, record.GetDeletedLocalPathIds().GetEnd()); - } - - if (record.HasLocalPathId()) { + } + + if (record.HasLocalPathId()) { ackPathId = ev->Get()->GetPathId(); - } - + } + if (record.HasMigratedLocalPathIds()) { ackPathId = TPathId(owner, record.GetMigratedLocalPathIds().GetEnd()); } const ui64 version = GetVersion(ackPathId); - Send(ev->Sender, new TSchemeBoardEvents::TEvUpdateAck(owner, generation, ackPathId, version), 0, ev->Cookie); - } - - void Handle(TSchemeBoardEvents::TEvHandshakeRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - const ui64 owner = record.GetOwner(); - const ui64 generation = record.GetGeneration(); - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvHandshakeRequest" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); - - TPopulatorInfo& info = Populators[owner]; - if (generation < info.PendingGeneration) { - SBR_LOG_E("Reject handshake from stale populator" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); - return; - } - - SBR_LOG_N("Successful handshake" - << ": self# " << SelfId() - << ", owner# " << owner - << ", generation# " << generation); - - info.PendingGeneration = generation; + Send(ev->Sender, new TSchemeBoardEvents::TEvUpdateAck(owner, generation, ackPathId, version), 0, ev->Cookie); + } + + void Handle(TSchemeBoardEvents::TEvHandshakeRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + const ui64 owner = record.GetOwner(); + const ui64 generation = record.GetGeneration(); + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvHandshakeRequest" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation); + + TPopulatorInfo& info = Populators[owner]; + if (generation < info.PendingGeneration) { + SBR_LOG_E("Reject handshake from stale populator" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation + << ", pending generation# " << info.PendingGeneration); + return; + } + + SBR_LOG_N("Successful handshake" + << ": self# " << SelfId() + << ", owner# " << owner + << ", generation# " << generation); + + info.PendingGeneration = generation; info.PopulatorActor = ev->Sender; - Send(ev->Sender, new TSchemeBoardEvents::TEvHandshakeResponse(owner, info.Generation), 0, ev->Cookie); - } - - void Handle(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { - auto& record = *ev->Get()->MutableRecord(); - - const ui64 owner = record.GetOwner(); - const ui64 generation = record.GetGeneration(); - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvUpdate" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", owner# " << owner - << ", generation# " << generation); + Send(ev->Sender, new TSchemeBoardEvents::TEvHandshakeResponse(owner, info.Generation), 0, ev->Cookie); + } + + void Handle(TSchemeBoardEvents::TEvUpdate::TPtr& ev) { + auto& record = *ev->Get()->MutableRecord(); + + const ui64 owner = record.GetOwner(); + const ui64 generation = record.GetGeneration(); + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvUpdate" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie + << ", owner# " << owner + << ", generation# " << generation); SBR_LOG_T("Message:\n" << ev->Get()->ToString().substr(0, 10000)); - - const auto populatorIt = Populators.find(owner); - if (populatorIt == Populators.end()) { - SBR_LOG_E("Reject update from unknown populator" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); - return; - } - if (generation != populatorIt->second.PendingGeneration) { - SBR_LOG_E("Reject update from stale populator" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << populatorIt->second.PendingGeneration); - return; - } - - if (record.HasDeletedLocalPathIds()) { - const TPathId begin(owner, record.GetDeletedLocalPathIds().GetBegin()); - const TPathId end(owner, record.GetDeletedLocalPathIds().GetEnd()); - SoftDeleteDescriptions(begin, end); - } - - if (!record.HasLocalPathId()) { - return AckUpdate(ev); - } - - const TString& path = record.GetPath(); + + const auto populatorIt = Populators.find(owner); + if (populatorIt == Populators.end()) { + SBR_LOG_E("Reject update from unknown populator" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation); + return; + } + if (generation != populatorIt->second.PendingGeneration) { + SBR_LOG_E("Reject update from stale populator" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation + << ", pending generation# " << populatorIt->second.PendingGeneration); + return; + } + + if (record.HasDeletedLocalPathIds()) { + const TPathId begin(owner, record.GetDeletedLocalPathIds().GetBegin()); + const TPathId end(owner, record.GetDeletedLocalPathIds().GetEnd()); + SoftDeleteDescriptions(begin, end); + } + + if (!record.HasLocalPathId()) { + return AckUpdate(ev); + } + + const TString& path = record.GetPath(); const TPathId pathId = ev->Get()->GetPathId(); - - SBR_LOG_N("Update description" - << ": self# " << SelfId() - << ", path# " << path + + SBR_LOG_N("Update description" + << ": self# " << SelfId() + << ", path# " << path << ", pathId# " << pathId << ", deletion# " << (record.GetIsDeletion() ? "true" : "false")); - - if (record.GetIsDeletion()) { - SoftDeleteDescription(pathId, true); + + if (record.GetIsDeletion()) { + SoftDeleteDescription(pathId, true); return AckUpdate(ev); } - if (TDescription* desc = Descriptions.FindPtr(pathId)) { - if (desc->IsExplicitlyDeleted()) { - SBR_LOG_N("Path was explicitly deleted, ignoring" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId); - - return AckUpdate(ev); - } - } - + if (TDescription* desc = Descriptions.FindPtr(pathId)) { + if (desc->IsExplicitlyDeleted()) { + SBR_LOG_N("Path was explicitly deleted, ignoring" + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId); + + return AckUpdate(ev); + } + } + TDescription* desc = Descriptions.FindPtr(path); if (!desc) { UpsertDescription(path, pathId, std::move(*record.MutableDescribeSchemeResult())); @@ -862,26 +862,26 @@ private: auto curPathId = desc->GetPathId(); - if (curPathId.OwnerId == pathId.OwnerId || !desc->IsFilled()) { + if (curPathId.OwnerId == pathId.OwnerId || !desc->IsFilled()) { if (curPathId > pathId) { - return AckUpdate(ev); - } - + return AckUpdate(ev); + } + if (curPathId < pathId) { SoftDeleteDescription(desc->GetPathId()); - Descriptions.DeleteIndex(path); - RelinkSubscribers(desc, path); - } - - UpsertDescription(path, pathId, std::move(*record.MutableDescribeSchemeResult())); + Descriptions.DeleteIndex(path); + RelinkSubscribers(desc, path); + } + + UpsertDescription(path, pathId, std::move(*record.MutableDescribeSchemeResult())); return AckUpdate(ev); - } - + } + Y_VERIFY_S(desc->IsFilled(), "desc :" - << ": path# " << desc->GetPath() - << ", pathId# " << desc->GetPathId() - << ", domainId# " << desc->GetDomainId() - << ", version# " << desc->GetVersion()); + << ": path# " << desc->GetPath() + << ", pathId# " << desc->GetPathId() + << ", domainId# " << desc->GetDomainId() + << ", version# " << desc->GetVersion()); auto curDomainId = desc->GetDomainId(); auto domainId = GetDomainId(record.GetDescribeSchemeResult()); @@ -892,24 +892,24 @@ private: auto abandonedSchemeShards = desc->GetAbandonedSchemeShardIds(); if (abandonedSchemeShards.contains(pathId.OwnerId)) { //TSS is ignored, present GSS reverted it SBR_LOG_N("Replace GSS by TSS description is rejected, GSS implicitly knows that TSS has been reverted" - ", but still inject description only by pathId for safe" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + ", but still inject description only by pathId for safe" + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); UpsertDescription(pathId, TDescription(this, path, pathId, std::move(*record.MutableDescribeSchemeResult()))); return AckUpdate(ev); } SBR_LOG_N("Replace GSS by TSS description" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); //unlick GSS desc by path Descriptions.DeleteIndex(path); RelinkSubscribers(desc, path); @@ -923,12 +923,12 @@ private: auto abandonedSchemeShards = GetAbandonedSchemeShardIds(record.GetDescribeSchemeResult()); if (abandonedSchemeShards.contains(curPathId.OwnerId)) { //GSS reverts TSS SBR_LOG_N("Replace TSS by GSS description, TSS was implicitly reverted by GSS" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); //unlick TSS desc by path Descriptions.DeleteIndex(path); RelinkSubscribers(desc, path); @@ -937,13 +937,13 @@ private: } SBR_LOG_N("Inject description only by pathId, it is update from GSS" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); - UpsertDescription(pathId, TDescription(this, path, pathId, std::move(*record.MutableDescribeSchemeResult()))); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); + UpsertDescription(pathId, TDescription(this, path, pathId, std::move(*record.MutableDescribeSchemeResult()))); return AckUpdate(ev); } @@ -977,313 +977,313 @@ private: return AckUpdate(ev); } else if (curDomainId < domainId) { SBR_LOG_N("Update description by newest path with newer domainId" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); Descriptions.DeleteIndex(path); RelinkSubscribers(desc, path); UpsertDescription(path, pathId, std::move(*record.MutableDescribeSchemeResult())); return AckUpdate(ev); } else { SBR_LOG_N("Totally ignore description, path with obsolete domainId" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); return AckUpdate(ev); } - Y_FAIL_S("Can't insert old description, no relation between obj" - << ": self# " << SelfId() - << ", path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); - } - - void Handle(TSchemeBoardEvents::TEvCommitRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - const ui64 owner = record.GetOwner(); - const ui64 generation = record.GetGeneration(); - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvCommitRequest" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); - - auto it = Populators.find(owner); - if (it == Populators.end()) { - SBR_LOG_E("Reject commit from unknown populator" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); - return; - } - - TPopulatorInfo& info = it->second; - if (generation != info.PendingGeneration) { - SBR_LOG_E("Reject commit from stale populator" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); - return; - } - - SBR_LOG_N("Commit generation" - << ": self# " << SelfId() - << ", owner# " << owner - << ", generation# " << generation); - - info.Generation = info.PendingGeneration; - - Send(ev->Sender, new TSchemeBoardEvents::TEvCommitResponse(owner, info.Generation), 0, ev->Cookie); - - for (const auto& [subscriber, id] : Subscribers) { - TDescription* desc = nullptr; - + Y_FAIL_S("Can't insert old description, no relation between obj" + << ": self# " << SelfId() + << ", path# " << path + << ", pathId# " << pathId + << ", domainId# " << domainId + << ", curPathId# " << curPathId + << ", curDomainId# " << curDomainId); + } + + void Handle(TSchemeBoardEvents::TEvCommitRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + const ui64 owner = record.GetOwner(); + const ui64 generation = record.GetGeneration(); + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvCommitRequest" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation); + + auto it = Populators.find(owner); + if (it == Populators.end()) { + SBR_LOG_E("Reject commit from unknown populator" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation); + return; + } + + TPopulatorInfo& info = it->second; + if (generation != info.PendingGeneration) { + SBR_LOG_E("Reject commit from stale populator" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", owner# " << owner + << ", generation# " << generation + << ", pending generation# " << info.PendingGeneration); + return; + } + + SBR_LOG_N("Commit generation" + << ": self# " << SelfId() + << ", owner# " << owner + << ", generation# " << generation); + + info.Generation = info.PendingGeneration; + + Send(ev->Sender, new TSchemeBoardEvents::TEvCommitResponse(owner, info.Generation), 0, ev->Cookie); + + for (const auto& [subscriber, id] : Subscribers) { + TDescription* desc = nullptr; + if (const TString* path = std::get_if<TString>(&id)) { - desc = Descriptions.FindPtr(*path); + desc = Descriptions.FindPtr(*path); } else if (const TPathId* pathId = std::get_if<TPathId>(&id)) { - desc = Descriptions.FindPtr(*pathId); - } - - Y_VERIFY(desc); - auto& info = desc->GetSubscriberInfo(subscriber); - - if (info.GetDomainOwnerId() != owner - || info.IsNotifiedStrongly() - || info.IsWaitForAck()) { - continue; - } - - auto notify = desc->BuildNotify(true); - info.EnqueueVersion(notify.Get()); - Send(subscriber, std::move(notify)); - } - } - - void Handle(TSchemeBoardEvents::TEvSubscribe::TPtr& ev) { - const auto& record = ev->Get()->Record; - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvSubscribe" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", record# " << record.ShortDebugString()); - - const ui64 domainOwnerId = record.GetDomainOwnerId(); - const auto& capabilities = record.GetCapabilities(); - - if (record.HasPath()) { - SubscribeBy(ev->Sender, record.GetPath(), domainOwnerId, capabilities); - } else { + desc = Descriptions.FindPtr(*pathId); + } + + Y_VERIFY(desc); + auto& info = desc->GetSubscriberInfo(subscriber); + + if (info.GetDomainOwnerId() != owner + || info.IsNotifiedStrongly() + || info.IsWaitForAck()) { + continue; + } + + auto notify = desc->BuildNotify(true); + info.EnqueueVersion(notify.Get()); + Send(subscriber, std::move(notify)); + } + } + + void Handle(TSchemeBoardEvents::TEvSubscribe::TPtr& ev) { + const auto& record = ev->Get()->Record; + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvSubscribe" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", record# " << record.ShortDebugString()); + + const ui64 domainOwnerId = record.GetDomainOwnerId(); + const auto& capabilities = record.GetCapabilities(); + + if (record.HasPath()) { + SubscribeBy(ev->Sender, record.GetPath(), domainOwnerId, capabilities); + } else { Y_VERIFY(record.HasPathOwnerId() && record.HasLocalPathId()); - SubscribeBy(ev->Sender, TPathId(record.GetPathOwnerId(), record.GetLocalPathId()), domainOwnerId, capabilities); - } - } - - void Handle(TSchemeBoardEvents::TEvUnsubscribe::TPtr& ev) { - const auto& record = ev->Get()->Record; - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvUnsubscribe" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", record# " << record.ShortDebugString()); - - if (record.HasPath()) { - UnsubscribeBy(ev->Sender, record.GetPath()); - } else { + SubscribeBy(ev->Sender, TPathId(record.GetPathOwnerId(), record.GetLocalPathId()), domainOwnerId, capabilities); + } + } + + void Handle(TSchemeBoardEvents::TEvUnsubscribe::TPtr& ev) { + const auto& record = ev->Get()->Record; + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvUnsubscribe" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", record# " << record.ShortDebugString()); + + if (record.HasPath()) { + UnsubscribeBy(ev->Sender, record.GetPath()); + } else { Y_VERIFY(record.HasPathOwnerId() && record.HasLocalPathId()); - UnsubscribeBy(ev->Sender, TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); - } - } - - void Handle(TSchemeBoardEvents::TEvNotifyAck::TPtr& ev) { - const auto& record = ev->Get()->Record; - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvNotifyAck" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", record# " << record.ShortDebugString()); - - auto it = Subscribers.find(ev->Sender); - if (it == Subscribers.end()) { - return; - } - - TDescription* desc = nullptr; - + UnsubscribeBy(ev->Sender, TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); + } + } + + void Handle(TSchemeBoardEvents::TEvNotifyAck::TPtr& ev) { + const auto& record = ev->Get()->Record; + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvNotifyAck" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", record# " << record.ShortDebugString()); + + auto it = Subscribers.find(ev->Sender); + if (it == Subscribers.end()) { + return; + } + + TDescription* desc = nullptr; + if (const TString* path = std::get_if<TString>(&it->second)) { - desc = Descriptions.FindPtr(*path); + desc = Descriptions.FindPtr(*path); } else if (const TPathId* pathId = std::get_if<TPathId>(&it->second)) { - desc = Descriptions.FindPtr(*pathId); - } - - Y_VERIFY(desc); - auto& info = desc->GetSubscriberInfo(ev->Sender); - - const ui64 version = record.GetVersion(); - if (!info.AckVersion(version)) { - return; - } - - if (version < desc->GetVersion()) { - auto notify = desc->BuildNotify(IsPopulatorCommited(info.GetDomainOwnerId())); - info.EnqueueVersion(notify.Get()); - Send(ev->Sender, std::move(notify)); - } - - if (auto cookie = info.ProcessSyncRequest()) { - Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(desc->GetVersion()), 0, *cookie); - } - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - SBR_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionRequest" - << ": self# " << SelfId() - << ", sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", record# " << record.ShortDebugString()); - - auto it = Subscribers.find(ev->Sender); - if (it == Subscribers.end()) { - // for backward compatability - ui64 version = 0; - - if (record.HasPath()) { - version = GetVersion(record.GetPath()); - } else { - version = GetVersion(TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); - } - - Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(version), 0, ev->Cookie); - return; - } - - TDescription* desc = nullptr; - + desc = Descriptions.FindPtr(*pathId); + } + + Y_VERIFY(desc); + auto& info = desc->GetSubscriberInfo(ev->Sender); + + const ui64 version = record.GetVersion(); + if (!info.AckVersion(version)) { + return; + } + + if (version < desc->GetVersion()) { + auto notify = desc->BuildNotify(IsPopulatorCommited(info.GetDomainOwnerId())); + info.EnqueueVersion(notify.Get()); + Send(ev->Sender, std::move(notify)); + } + + if (auto cookie = info.ProcessSyncRequest()) { + Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(desc->GetVersion()), 0, *cookie); + } + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + SBR_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionRequest" + << ": self# " << SelfId() + << ", sender# " << ev->Sender + << ", cookie# " << ev->Cookie + << ", record# " << record.ShortDebugString()); + + auto it = Subscribers.find(ev->Sender); + if (it == Subscribers.end()) { + // for backward compatability + ui64 version = 0; + + if (record.HasPath()) { + version = GetVersion(record.GetPath()); + } else { + version = GetVersion(TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); + } + + Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(version), 0, ev->Cookie); + return; + } + + TDescription* desc = nullptr; + if (const TString* path = std::get_if<TString>(&it->second)) { - desc = Descriptions.FindPtr(*path); + desc = Descriptions.FindPtr(*path); } else if (const TPathId* pathId = std::get_if<TPathId>(&it->second)) { - desc = Descriptions.FindPtr(*pathId); - } - - Y_VERIFY(desc); - auto& info = desc->GetSubscriberInfo(ev->Sender); - - if (!info.EnqueueSyncRequest(ev->Cookie) || info.IsWaitForAck()) { - return; - } - - auto cookie = info.ProcessSyncRequest(); - Y_VERIFY(cookie && *cookie == ev->Cookie); - - Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(desc->GetVersion()), 0, *cookie); - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); - - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); - auto& record = *response->Record.MutableReplicaResponse(); - - for (const auto& [owner, populator] : Populators) { - auto& info = *record.AddPopulators(); - - info.SetOwner(owner); - info.SetGeneration(populator.Generation); - info.SetPendingGeneration(populator.PendingGeneration); - ActorIdToProto(populator.PopulatorActor, info.MutableActorId()); - - if (record.PopulatorsSize() >= limit) { - response->SetTruncated(); - break; - } - } - - record.MutableDescriptions()->SetTotalCount(Descriptions.Size()); - record.MutableDescriptions()->SetByPathCount(Descriptions.GetPrimaryIndex().size()); - record.MutableDescriptions()->SetByPathIdCount(Descriptions.GetSecondaryIndex().size()); - - for (const auto& [subscriber, id] : Subscribers) { - auto& info = *record.AddSubscribers(); - - ActorIdToProto(subscriber, info.MutableActorId()); + desc = Descriptions.FindPtr(*pathId); + } + + Y_VERIFY(desc); + auto& info = desc->GetSubscriberInfo(ev->Sender); + + if (!info.EnqueueSyncRequest(ev->Cookie) || info.IsWaitForAck()) { + return; + } + + auto cookie = info.ProcessSyncRequest(); + Y_VERIFY(cookie && *cookie == ev->Cookie); + + Send(ev->Sender, new TSchemeBoardEvents::TEvSyncVersionResponse(desc->GetVersion()), 0, *cookie); + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + const auto limit = ev->Get()->Record.GetLimitRepeatedFields(); + + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(SelfId(), ActorActivityType()); + auto& record = *response->Record.MutableReplicaResponse(); + + for (const auto& [owner, populator] : Populators) { + auto& info = *record.AddPopulators(); + + info.SetOwner(owner); + info.SetGeneration(populator.Generation); + info.SetPendingGeneration(populator.PendingGeneration); + ActorIdToProto(populator.PopulatorActor, info.MutableActorId()); + + if (record.PopulatorsSize() >= limit) { + response->SetTruncated(); + break; + } + } + + record.MutableDescriptions()->SetTotalCount(Descriptions.Size()); + record.MutableDescriptions()->SetByPathCount(Descriptions.GetPrimaryIndex().size()); + record.MutableDescriptions()->SetByPathIdCount(Descriptions.GetSecondaryIndex().size()); + + for (const auto& [subscriber, id] : Subscribers) { + auto& info = *record.AddSubscribers(); + + ActorIdToProto(subscriber, info.MutableActorId()); if (const TString* path = std::get_if<TString>(&id)) { - info.SetPath(*path); + info.SetPath(*path); } else if (const TPathId* pathId = std::get_if<TPathId>(&id)) { - info.MutablePathId()->SetOwnerId(pathId->OwnerId); - info.MutablePathId()->SetLocalPathId(pathId->LocalPathId); - } - - if (record.SubscribersSize() >= limit) { - response->SetTruncated(); - break; - } - } - - Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { - const auto& record = ev->Get()->Record; - - TDescription* desc = nullptr; - if (record.HasPath()) { - desc = Descriptions.FindPtr(record.GetPath()); - } else if (record.HasPathId()) { - desc = Descriptions.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); - } - - TString json; - if (desc) { - using namespace google::protobuf::util; - - JsonPrintOptions opts; - opts.preserve_proto_field_names = true; - MessageToJsonString(desc->GetProto(), &json, opts); - } else { - json = "{}"; - } - - Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(json), 0, ev->Cookie); - } - - void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - const ui32 nodeId = ev->Get()->NodeId; - - SBR_LOG_D("Handle TEvInterconnect::TEvNodeDisconnected" - << ": self# " << SelfId() - << ", nodeId# " << nodeId); - + info.MutablePathId()->SetOwnerId(pathId->OwnerId); + info.MutablePathId()->SetLocalPathId(pathId->LocalPathId); + } + + if (record.SubscribersSize() >= limit) { + response->SetTruncated(); + break; + } + } + + Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void Handle(TSchemeBoardMonEvents::TEvDescribeRequest::TPtr& ev) { + const auto& record = ev->Get()->Record; + + TDescription* desc = nullptr; + if (record.HasPath()) { + desc = Descriptions.FindPtr(record.GetPath()); + } else if (record.HasPathId()) { + desc = Descriptions.FindPtr(TPathId(record.GetPathId().GetOwnerId(), record.GetPathId().GetLocalPathId())); + } + + TString json; + if (desc) { + using namespace google::protobuf::util; + + JsonPrintOptions opts; + opts.preserve_proto_field_names = true; + MessageToJsonString(desc->GetProto(), &json, opts); + } else { + json = "{}"; + } + + Send(ev->Sender, new TSchemeBoardMonEvents::TEvDescribeResponse(json), 0, ev->Cookie); + } + + void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + const ui32 nodeId = ev->Get()->NodeId; + + SBR_LOG_D("Handle TEvInterconnect::TEvNodeDisconnected" + << ": self# " << SelfId() + << ", nodeId# " << nodeId); + auto it = Subscribers.lower_bound(TActorId(nodeId, 0, 0, 0)); - while (it != Subscribers.end() && it->first.NodeId() == nodeId) { + while (it != Subscribers.end() && it->first.NodeId() == nodeId) { const TActorId subscriber = it->first; - const auto id = it->second; - ++it; - + const auto id = it->second; + ++it; + if (const TString* path = std::get_if<TString>(&id)) { - Unsubscribe(subscriber, *path); + Unsubscribe(subscriber, *path); } else if (const TPathId* pathId = std::get_if<TPathId>(&id)) { - Unsubscribe(subscriber, *pathId); - } - } - - Send(MakeInterconnectProxyId(nodeId), new TEvents::TEvUnsubscribe()); - } - + Unsubscribe(subscriber, *pathId); + } + } + + Send(MakeInterconnectProxyId(nodeId), new TEvents::TEvUnsubscribe()); + } + void PassAway() override { for (auto &xpair : Populators) { if (const TActorId populator = xpair.second.PopulatorActor) { @@ -1295,48 +1295,48 @@ private: Send(xpair.first, new TEvStateStorage::TEvReplicaShutdown()); } - TMonitorableActor::PassAway(); + TMonitorableActor::PassAway(); } - -public: + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR; - } - - void Bootstrap() { - TMonitorableActor::Bootstrap(); - Become(&TThis::StateWork); - } - + } + + void Bootstrap() { + TMonitorableActor::Bootstrap(); + Become(&TThis::StateWork); + } + STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvHandshakeRequest, Handle); - hFunc(TSchemeBoardEvents::TEvUpdate, Handle); - hFunc(TSchemeBoardEvents::TEvCommitRequest, Handle); - hFunc(TSchemeBoardEvents::TEvSubscribe, Handle); - hFunc(TSchemeBoardEvents::TEvUnsubscribe, Handle); - hFunc(TSchemeBoardEvents::TEvNotifyAck, Handle); - hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); - - hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvHandshakeRequest, Handle); + hFunc(TSchemeBoardEvents::TEvUpdate, Handle); + hFunc(TSchemeBoardEvents::TEvCommitRequest, Handle); + hFunc(TSchemeBoardEvents::TEvSubscribe, Handle); + hFunc(TSchemeBoardEvents::TEvUnsubscribe, Handle); + hFunc(TSchemeBoardEvents::TEvNotifyAck, Handle); + hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + hFunc(TSchemeBoardMonEvents::TEvDescribeRequest, Handle); + + hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - -private: - THashMap<ui64, TPopulatorInfo> Populators; - TDoubleIndexedMap<TString, TPathId, TDescription, TMerger, THashMap, TMap> Descriptions; + } + } + +private: + THashMap<ui64, TPopulatorInfo> Populators; + TDoubleIndexedMap<TString, TPathId, TDescription, TMerger, THashMap, TMap> Descriptions; TMap<TActorId, std::variant<TString, TPathId>, TActorId::TOrderedCmp> Subscribers; - -}; // TReplica - -} // NSchemeBoard - -IActor* CreateSchemeBoardReplica(const TIntrusivePtr<TStateStorageInfo>&, ui32) { - return new NSchemeBoard::TReplica(); -} - -} // NKikimr + +}; // TReplica + +} // NSchemeBoard + +IActor* CreateSchemeBoardReplica(const TIntrusivePtr<TStateStorageInfo>&, ui32) { + return new NSchemeBoard::TReplica(); +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/replica.h b/ydb/core/tx/scheme_board/replica.h index f75e0f0b454..a626fea2a1e 100644 --- a/ydb/core/tx/scheme_board/replica.h +++ b/ydb/core/tx/scheme_board/replica.h @@ -1,21 +1,21 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/base/statestorage.h> - -namespace NKikimr { - -// same as MakeStateStorageReplicaID + +namespace NKikimr { + +// same as MakeStateStorageReplicaID inline TActorId MakeSchemeBoardReplicaID( - const ui32 node, - const ui64 stateStorageGroup, - const ui32 replicaIndex -) { - char x[12] = { 's', 'b', 's' }; - x[3] = (char)stateStorageGroup; - memcpy(x + 5, &replicaIndex, sizeof(ui32)); + const ui32 node, + const ui64 stateStorageGroup, + const ui32 replicaIndex +) { + char x[12] = { 's', 'b', 's' }; + x[3] = (char)stateStorageGroup; + memcpy(x + 5, &replicaIndex, sizeof(ui32)); return TActorId(node, TStringBuf(x, 12)); -} - -} // NKikimr +} + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/replica_ut.cpp b/ydb/core/tx/scheme_board/replica_ut.cpp index 78ec30ad0dc..10113c7bcce 100644 --- a/ydb/core/tx/scheme_board/replica_ut.cpp +++ b/ydb/core/tx/scheme_board/replica_ut.cpp @@ -1,554 +1,554 @@ -#include "events.h" -#include "replica.h" -#include "ut_helpers.h" - +#include "events.h" +#include "replica.h" +#include "ut_helpers.h" + #include <ydb/core/base/pathid.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/testlib/basics/appdata.h> - + #include <library/cpp/actors/core/log.h> #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NSchemeBoard { - -namespace { - - TActorId CreateReplica(TTestActorRuntimeBase& runtime) { - const auto replica = runtime.Register(CreateSchemeBoardReplica(TIntrusivePtr<TStateStorageInfo>(), 0)); - runtime.EnableScheduleForActor(replica, true); - - // wait until replica is ready - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&replica](IEventHandle& ev) { - return ev.Recipient == replica && ev.GetTypeRewrite() == TEvents::TSystem::Bootstrap; - }); - runtime.DispatchEvents(opts); - - return replica; - } - -} // anonymous - -class TReplicaTest: public NUnitTest::TTestBase { -public: - void SetUp() override { - Context = MakeHolder<TTestContext>(); - Context->Initialize(TAppPrepare().Unwrap()); - Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NLog::PRI_DEBUG); - - Replica = CreateReplica(*Context); - } - - void TearDown() override { - Context.Reset(); - } - - UNIT_TEST_SUITE(TReplicaTest); - UNIT_TEST(Handshake); - UNIT_TEST(HandshakeWithStaleGeneration); - UNIT_TEST(Commit); - UNIT_TEST(CommitWithoutHandshake); - UNIT_TEST(CommitWithStaleGeneration); - UNIT_TEST(Update); - UNIT_TEST(Delete); - UNIT_TEST(UpdateWithoutHandshake); - UNIT_TEST(UpdateWithStaleGeneration); - UNIT_TEST(Subscribe); - UNIT_TEST(SubscribeUnknownPath); - UNIT_TEST(Unsubscribe); - UNIT_TEST(UnsubscribeUnknownPath); - UNIT_TEST(DoubleUnsubscribe); - UNIT_TEST(UnsubscribeWithoutSubscribe); - UNIT_TEST(Merge); - UNIT_TEST(DoubleDelete); - UNIT_TEST(SyncVersion); - UNIT_TEST(IdempotencyUpdatesAliveSubscriber); - UNIT_TEST(IdempotencyUpdatesWithoutSubscribers); - UNIT_TEST(IdempotencyUpdatesVariant2); - UNIT_TEST(AckNotifications); - UNIT_TEST(AckNotificationsUponPathRecreation); - UNIT_TEST(StrongNotificationAfterCommit); - UNIT_TEST_SUITE_END(); - - void Handshake(); - void HandshakeWithStaleGeneration(); - void Commit(); - void CommitWithoutHandshake(); - void CommitWithStaleGeneration(); - void Update(); - void Delete(); - void UpdateWithoutHandshake(); - void UpdateWithStaleGeneration(); - void Subscribe(); - void SubscribeUnknownPath(); - void Unsubscribe(); - void UnsubscribeUnknownPath(); - void DoubleUnsubscribe(); - void UnsubscribeWithoutSubscribe(); - void Merge(); - void DoubleDelete(); - void SyncVersion(); - void IdempotencyUpdates(bool aliveSubscriber); - void IdempotencyUpdatesAliveSubscriber(); - void IdempotencyUpdatesWithoutSubscribers(); - void IdempotencyUpdatesVariant2(); - void AckNotifications(); - void AckNotificationsUponPathRecreation(); - void StrongNotificationAfterCommit(); - -private: - THolder<TTestContext> Context; + +namespace NKikimr { +namespace NSchemeBoard { + +namespace { + + TActorId CreateReplica(TTestActorRuntimeBase& runtime) { + const auto replica = runtime.Register(CreateSchemeBoardReplica(TIntrusivePtr<TStateStorageInfo>(), 0)); + runtime.EnableScheduleForActor(replica, true); + + // wait until replica is ready + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&replica](IEventHandle& ev) { + return ev.Recipient == replica && ev.GetTypeRewrite() == TEvents::TSystem::Bootstrap; + }); + runtime.DispatchEvents(opts); + + return replica; + } + +} // anonymous + +class TReplicaTest: public NUnitTest::TTestBase { +public: + void SetUp() override { + Context = MakeHolder<TTestContext>(); + Context->Initialize(TAppPrepare().Unwrap()); + Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NLog::PRI_DEBUG); + + Replica = CreateReplica(*Context); + } + + void TearDown() override { + Context.Reset(); + } + + UNIT_TEST_SUITE(TReplicaTest); + UNIT_TEST(Handshake); + UNIT_TEST(HandshakeWithStaleGeneration); + UNIT_TEST(Commit); + UNIT_TEST(CommitWithoutHandshake); + UNIT_TEST(CommitWithStaleGeneration); + UNIT_TEST(Update); + UNIT_TEST(Delete); + UNIT_TEST(UpdateWithoutHandshake); + UNIT_TEST(UpdateWithStaleGeneration); + UNIT_TEST(Subscribe); + UNIT_TEST(SubscribeUnknownPath); + UNIT_TEST(Unsubscribe); + UNIT_TEST(UnsubscribeUnknownPath); + UNIT_TEST(DoubleUnsubscribe); + UNIT_TEST(UnsubscribeWithoutSubscribe); + UNIT_TEST(Merge); + UNIT_TEST(DoubleDelete); + UNIT_TEST(SyncVersion); + UNIT_TEST(IdempotencyUpdatesAliveSubscriber); + UNIT_TEST(IdempotencyUpdatesWithoutSubscribers); + UNIT_TEST(IdempotencyUpdatesVariant2); + UNIT_TEST(AckNotifications); + UNIT_TEST(AckNotificationsUponPathRecreation); + UNIT_TEST(StrongNotificationAfterCommit); + UNIT_TEST_SUITE_END(); + + void Handshake(); + void HandshakeWithStaleGeneration(); + void Commit(); + void CommitWithoutHandshake(); + void CommitWithStaleGeneration(); + void Update(); + void Delete(); + void UpdateWithoutHandshake(); + void UpdateWithStaleGeneration(); + void Subscribe(); + void SubscribeUnknownPath(); + void Unsubscribe(); + void UnsubscribeUnknownPath(); + void DoubleUnsubscribe(); + void UnsubscribeWithoutSubscribe(); + void Merge(); + void DoubleDelete(); + void SyncVersion(); + void IdempotencyUpdates(bool aliveSubscriber); + void IdempotencyUpdatesAliveSubscriber(); + void IdempotencyUpdatesWithoutSubscribers(); + void IdempotencyUpdatesVariant2(); + void AckNotifications(); + void AckNotificationsUponPathRecreation(); + void StrongNotificationAfterCommit(); + +private: + THolder<TTestContext> Context; TActorId Replica; - -}; // TReplicaTest - -UNIT_TEST_SUITE_REGISTRATION(TReplicaTest); - -void TReplicaTest::Handshake() { - auto ev = Context->HandshakeReplica(Replica, Context->AllocateEdgeActor(), 1, 1); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); - UNIT_ASSERT_VALUES_EQUAL(0, ev->Get()->Record.GetGeneration()); -} - -void TReplicaTest::HandshakeWithStaleGeneration() { + +}; // TReplicaTest + +UNIT_TEST_SUITE_REGISTRATION(TReplicaTest); + +void TReplicaTest::Handshake() { + auto ev = Context->HandshakeReplica(Replica, Context->AllocateEdgeActor(), 1, 1); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); + UNIT_ASSERT_VALUES_EQUAL(0, ev->Get()->Record.GetGeneration()); +} + +void TReplicaTest::HandshakeWithStaleGeneration() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 2); - Context->HandshakeReplica(Replica, edge, 1, 1, false); - - ui64 counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvHandshakeResponse>(); - UNIT_ASSERT_VALUES_EQUAL(0, counter); -} - -void TReplicaTest::Commit() { + + Context->HandshakeReplica(Replica, edge, 1, 2); + Context->HandshakeReplica(Replica, edge, 1, 1, false); + + ui64 counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvHandshakeResponse>(); + UNIT_ASSERT_VALUES_EQUAL(0, counter); +} + +void TReplicaTest::Commit() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - Context->CommitReplica(Replica, edge, 1, 1); - - auto ev = Context->HandshakeReplica(Replica, edge, 1, 2); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetGeneration()); -} - -void TReplicaTest::CommitWithoutHandshake() { + + Context->HandshakeReplica(Replica, edge, 1, 1); + Context->CommitReplica(Replica, edge, 1, 1); + + auto ev = Context->HandshakeReplica(Replica, edge, 1, 2); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetGeneration()); +} + +void TReplicaTest::CommitWithoutHandshake() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->CommitReplica(Replica, edge, 1, 1); - auto ev = Context->HandshakeReplica(Replica, edge, 1, 1); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); - UNIT_ASSERT_VALUES_EQUAL(0, ev->Get()->Record.GetGeneration()); -} - -void TReplicaTest::CommitWithStaleGeneration() { + + Context->CommitReplica(Replica, edge, 1, 1); + auto ev = Context->HandshakeReplica(Replica, edge, 1, 1); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); + UNIT_ASSERT_VALUES_EQUAL(0, ev->Get()->Record.GetGeneration()); +} + +void TReplicaTest::CommitWithStaleGeneration() { const TActorId edgeA = Context->AllocateEdgeActor(); const TActorId edgeB = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edgeA, 1, 0); - Context->HandshakeReplica(Replica, edgeB, 1, 1); - - Context->CommitReplica(Replica, edgeB, 1, 1); - Context->CommitReplica(Replica, edgeA, 1, 0); - - auto ev = Context->HandshakeReplica(Replica, edgeA, 1, 2); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetGeneration()); -} - -void TReplicaTest::Update() { + + Context->HandshakeReplica(Replica, edgeA, 1, 0); + Context->HandshakeReplica(Replica, edgeB, 1, 1); + + Context->CommitReplica(Replica, edgeB, 1, 1); + Context->CommitReplica(Replica, edgeA, 1, 0); + + auto ev = Context->HandshakeReplica(Replica, edgeA, 1, 2); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetOwner()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->Record.GetGeneration()); +} + +void TReplicaTest::Update() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); + + Context->HandshakeReplica(Replica, edge, 1, 1); Context->Send(Replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)), 1, 1)); - - auto checks = [](const auto& record) { - UNIT_ASSERT_VALUES_EQUAL(1, record.GetPathOwnerId()); - UNIT_ASSERT_VALUES_EQUAL("path", record.GetPath()); - UNIT_ASSERT_VALUES_EQUAL(1, record.GetLocalPathId()); - UNIT_ASSERT_VALUES_EQUAL(false, record.GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(1, record.GetVersion()); - }; - - { - auto ev = Context->SubscribeReplica(Replica, edge, "path"); - checks(ev->Get()->GetRecord()); - Context->UnsubscribeReplica(Replica, edge, "path"); - } - - { - auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); - checks(ev->Get()->GetRecord()); - Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); - } -} - -void TReplicaTest::Delete() { + + auto checks = [](const auto& record) { + UNIT_ASSERT_VALUES_EQUAL(1, record.GetPathOwnerId()); + UNIT_ASSERT_VALUES_EQUAL("path", record.GetPath()); + UNIT_ASSERT_VALUES_EQUAL(1, record.GetLocalPathId()); + UNIT_ASSERT_VALUES_EQUAL(false, record.GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(1, record.GetVersion()); + }; + + { + auto ev = Context->SubscribeReplica(Replica, edge, "path"); + checks(ev->Get()->GetRecord()); + Context->UnsubscribeReplica(Replica, edge, "path"); + } + + { + auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); + checks(ev->Get()->GetRecord()); + Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); + } +} + +void TReplicaTest::Delete() { auto describe = GenerateDescribe("path", TPathId(42, 1)); const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); - - { + + Context->HandshakeReplica(Replica, edge, 1, 1); + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); + + { auto ev = Context->SubscribeReplica(Replica, Context->AllocateEdgeActor(), "path"); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); - } - - { + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); + } + + { auto ev = Context->SubscribeReplica(Replica, Context->AllocateEdgeActor(), TPathId(42, 1)); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); - } - - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); - - { + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); + } + + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); + + { auto ev = Context->SubscribeReplica(Replica, Context->AllocateEdgeActor(), "path"); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); } { auto ev = Context->SubscribeReplica(Replica, Context->AllocateEdgeActor(), TPathId(42, 1)); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); - } - - { + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); + } + + { auto ev = Context->SubscribeReplica(Replica, Context->AllocateEdgeActor(), "path"); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); - } - -} - -void TReplicaTest::UpdateWithoutHandshake() { + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(Max<ui64>(), ev->Get()->GetRecord().GetVersion()); + } + +} + +void TReplicaTest::UpdateWithoutHandshake() { const TActorId edge = Context->AllocateEdgeActor(); - + Context->Send(Replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)), 1, 1)); - - { - auto ev = Context->SubscribeReplica(Replica, edge, "path"); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - Context->UnsubscribeReplica(Replica, edge, "path"); - } - - { - auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); - } -} - -void TReplicaTest::UpdateWithStaleGeneration() { + + { + auto ev = Context->SubscribeReplica(Replica, edge, "path"); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + Context->UnsubscribeReplica(Replica, edge, "path"); + } + + { + auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); + } +} + +void TReplicaTest::UpdateWithStaleGeneration() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); + + Context->HandshakeReplica(Replica, edge, 1, 1); Context->Send(Replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)), 1, 0)); - - { - auto ev = Context->SubscribeReplica(Replica, edge, "path"); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - Context->UnsubscribeReplica(Replica, edge, "path"); - } - - { - auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); - } -} - -void TReplicaTest::Subscribe() { + + { + auto ev = Context->SubscribeReplica(Replica, edge, "path"); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + Context->UnsubscribeReplica(Replica, edge, "path"); + } + + { + auto ev = Context->SubscribeReplica(Replica, edge, TPathId(1, 1)); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + Context->UnsubscribeReplica(Replica, edge, TPathId(1, 1)); + } +} + +void TReplicaTest::Subscribe() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); - - Context->SubscribeReplica(Replica, edge, "path", false); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); - } - - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - } -} - -void TReplicaTest::SubscribeUnknownPath() { + + Context->HandshakeReplica(Replica, edge, 1, 1); + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); + + Context->SubscribeReplica(Replica, edge, "path", false); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); + } + + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + } +} + +void TReplicaTest::SubscribeUnknownPath() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->SubscribeReplica(Replica, edge, "path", false); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - } -} - -void TReplicaTest::Unsubscribe() { + + Context->SubscribeReplica(Replica, edge, "path", false); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(edge); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + } +} + +void TReplicaTest::Unsubscribe() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId edge = Context->AllocateEdgeActor(); - + const TActorId subscriberA = Context->AllocateEdgeActor(); const TActorId subscriberB = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - - Context->SubscribeReplica(Replica, subscriberA, "path"); - Context->SubscribeReplica(Replica, subscriberB, "path"); - - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); - ui64 counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotify>(); - UNIT_ASSERT_VALUES_EQUAL(2, counter); - - Context->UnsubscribeReplica(Replica, subscriberA, "path"); - - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); - counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotify>(); - UNIT_ASSERT_VALUES_EQUAL(1, counter); -} - -void TReplicaTest::UnsubscribeUnknownPath() { + + Context->HandshakeReplica(Replica, edge, 1, 1); + + Context->SubscribeReplica(Replica, subscriberA, "path"); + Context->SubscribeReplica(Replica, subscriberB, "path"); + + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); + ui64 counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotify>(); + UNIT_ASSERT_VALUES_EQUAL(2, counter); + + Context->UnsubscribeReplica(Replica, subscriberA, "path"); + + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1, true)); + counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotify>(); + UNIT_ASSERT_VALUES_EQUAL(1, counter); +} + +void TReplicaTest::UnsubscribeUnknownPath() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->UnsubscribeReplica(Replica, edge, "path"); -} - -void TReplicaTest::DoubleUnsubscribe() { + + Context->UnsubscribeReplica(Replica, edge, "path"); +} + +void TReplicaTest::DoubleUnsubscribe() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); - - auto ev = Context->SubscribeReplica(Replica, edge, "path", true); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); - - Context->UnsubscribeReplica(Replica, edge, "path"); - Context->UnsubscribeReplica(Replica, edge, "path"); -} - -void TReplicaTest::UnsubscribeWithoutSubscribe() { + + Context->HandshakeReplica(Replica, edge, 1, 1); + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); + + auto ev = Context->SubscribeReplica(Replica, edge, "path", true); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetIsDeletion()); + + Context->UnsubscribeReplica(Replica, edge, "path"); + Context->UnsubscribeReplica(Replica, edge, "path"); +} + +void TReplicaTest::UnsubscribeWithoutSubscribe() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId edge = Context->AllocateEdgeActor(); - - Context->HandshakeReplica(Replica, edge, 1, 1); - Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); - - Context->UnsubscribeReplica(Replica, edge, "path"); -} - -void TReplicaTest::Merge() { + + Context->HandshakeReplica(Replica, edge, 1, 1); + Context->Send(Replica, edge, GenerateUpdate(describe, 1, 1)); + + Context->UnsubscribeReplica(Replica, edge, "path"); +} + +void TReplicaTest::Merge() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId populator = Context->AllocateEdgeActor(); const TActorId subscriberA = Context->AllocateEdgeActor(); const TActorId subscriberB = Context->AllocateEdgeActor(); - - Context->SubscribeReplica(Replica, subscriberA, "path"); - Context->SubscribeReplica(Replica, subscriberB, TPathId(1, 1)); - - Context->HandshakeReplica(Replica, populator); - - Context->Send(Replica, populator, GenerateUpdate(describe)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberB); - - Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberB); -} - -void TReplicaTest::DoubleDelete() { + + Context->SubscribeReplica(Replica, subscriberA, "path"); + Context->SubscribeReplica(Replica, subscriberB, TPathId(1, 1)); + + Context->HandshakeReplica(Replica, populator); + + Context->Send(Replica, populator, GenerateUpdate(describe)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberB); + + Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberB); +} + +void TReplicaTest::DoubleDelete() { auto describe = GenerateDescribe("path", TPathId(1, 1)); - + const TActorId populator = Context->AllocateEdgeActor(); - Context->HandshakeReplica(Replica, populator); - + Context->HandshakeReplica(Replica, populator); + const TActorId subscriberA = Context->AllocateEdgeActor(); - Context->SubscribeReplica(Replica, subscriberA, "path"); - - Context->Send(Replica, populator, GenerateUpdate(describe)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); - - Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); - + Context->SubscribeReplica(Replica, subscriberA, "path"); + + Context->Send(Replica, populator, GenerateUpdate(describe)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); + + Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriberA); + const TActorId subscriberB = Context->AllocateEdgeActor(); - Context->SubscribeReplica(Replica, subscriberB, "path"); - - Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); -} - -void TReplicaTest::SyncVersion() { + Context->SubscribeReplica(Replica, subscriberB, "path"); + + Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); +} + +void TReplicaTest::SyncVersion() { const TActorId edge = Context->AllocateEdgeActor(); - const ui64 version = 100500; - - Context->HandshakeReplica(Replica, edge); + const ui64 version = 100500; + + Context->HandshakeReplica(Replica, edge); Context->Send(Replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1), version))); - - Context->SubscribeReplica(Replica, edge, "path"); - Context->Send(Replica, edge, new TSchemeBoardEvents::TEvSyncVersionRequest("path"), 0, 1); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncVersionResponse>(edge); - - UNIT_ASSERT_VALUES_EQUAL(version, ev->Get()->Record.GetVersion()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->Record.GetPartial()); -} - -void TReplicaTest::IdempotencyUpdates(bool aliveSubscriber) { - auto describeA = GenerateDescribe("path", TPathId(1, 1)); - auto describeB = GenerateDescribe("path", TPathId(1, 2)); // same path but another path id - + + Context->SubscribeReplica(Replica, edge, "path"); + Context->Send(Replica, edge, new TSchemeBoardEvents::TEvSyncVersionRequest("path"), 0, 1); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncVersionResponse>(edge); + + UNIT_ASSERT_VALUES_EQUAL(version, ev->Get()->Record.GetVersion()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->Record.GetPartial()); +} + +void TReplicaTest::IdempotencyUpdates(bool aliveSubscriber) { + auto describeA = GenerateDescribe("path", TPathId(1, 1)); + auto describeB = GenerateDescribe("path", TPathId(1, 2)); // same path but another path id + const TActorId populator = Context->AllocateEdgeActor(); - Context->HandshakeReplica(Replica, populator); - + Context->HandshakeReplica(Replica, populator); + const TActorId subscriberA = Context->AllocateEdgeActor(); - Context->SubscribeReplica(Replica, subscriberA, TPathId(1, 1)); - - Context->Send(Replica, populator, GenerateUpdate(describeA)); - Context->Send(Replica, populator, GenerateUpdate(describeA, 1, 1, true)); - - if (!aliveSubscriber) { - Context->UnsubscribeReplica(Replica, subscriberA, TPathId(1, 1)); - } - - Context->Send(Replica, populator, GenerateUpdate(describeA)); - Context->Send(Replica, populator, GenerateUpdate(describeB)); - + Context->SubscribeReplica(Replica, subscriberA, TPathId(1, 1)); + + Context->Send(Replica, populator, GenerateUpdate(describeA)); + Context->Send(Replica, populator, GenerateUpdate(describeA, 1, 1, true)); + + if (!aliveSubscriber) { + Context->UnsubscribeReplica(Replica, subscriberA, TPathId(1, 1)); + } + + Context->Send(Replica, populator, GenerateUpdate(describeA)); + Context->Send(Replica, populator, GenerateUpdate(describeB)); + const TActorId subscriberB = Context->AllocateEdgeActor(); - Context->SubscribeReplica(Replica, subscriberB, TPathId(1, 2)); -} - -void TReplicaTest::IdempotencyUpdatesAliveSubscriber() { - IdempotencyUpdates(true); -} - -void TReplicaTest::IdempotencyUpdatesWithoutSubscribers() { - IdempotencyUpdates(false); -} - -void TReplicaTest::IdempotencyUpdatesVariant2() { - auto describeA = GenerateDescribe("path", TPathId(1, 1)); - auto describeB = GenerateDescribe("path", TPathId(1, 2)); // same path but another path id - + Context->SubscribeReplica(Replica, subscriberB, TPathId(1, 2)); +} + +void TReplicaTest::IdempotencyUpdatesAliveSubscriber() { + IdempotencyUpdates(true); +} + +void TReplicaTest::IdempotencyUpdatesWithoutSubscribers() { + IdempotencyUpdates(false); +} + +void TReplicaTest::IdempotencyUpdatesVariant2() { + auto describeA = GenerateDescribe("path", TPathId(1, 1)); + auto describeB = GenerateDescribe("path", TPathId(1, 2)); // same path but another path id + const TActorId populator = Context->AllocateEdgeActor(); - Context->HandshakeReplica(Replica, populator); - - Context->Send(Replica, populator, GenerateUpdate(describeA)); - Context->Send(Replica, populator, GenerateUpdate(describeB)); - Context->Send(Replica, populator, GenerateUpdate(describeB, 1, 1, true)); - Context->Send(Replica, populator, GenerateUpdate(describeA)); - Context->Send(Replica, populator, GenerateUpdate(describeB)); -} - -void TReplicaTest::AckNotifications() { - auto describe = GenerateDescribe("path", TPathId(1, 1)); - + Context->HandshakeReplica(Replica, populator); + + Context->Send(Replica, populator, GenerateUpdate(describeA)); + Context->Send(Replica, populator, GenerateUpdate(describeB)); + Context->Send(Replica, populator, GenerateUpdate(describeB, 1, 1, true)); + Context->Send(Replica, populator, GenerateUpdate(describeA)); + Context->Send(Replica, populator, GenerateUpdate(describeB)); +} + +void TReplicaTest::AckNotifications() { + auto describe = GenerateDescribe("path", TPathId(1, 1)); + const TActorId populator = Context->AllocateEdgeActor(); const TActorId subscriber = Context->AllocateEdgeActor(); - - NKikimrSchemeBoard::TEvSubscribe::TCapabilities capabilities; - capabilities.SetAckNotifications(true); - Context->SubscribeReplica(Replica, subscriber, "path", true, 0, capabilities); - - Context->HandshakeReplica(Replica, populator); - - Context->Send(Replica, populator, GenerateUpdate(describe)); - Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(0)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); - - Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); - Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(1)); - Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); -} - -void TReplicaTest::AckNotificationsUponPathRecreation() { - const TActorId populator = Context->AllocateEdgeActor(); - Context->HandshakeReplica(Replica, populator); - - // initial path version is 2 - Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1), 2))); - - const TActorId subscriber = Context->AllocateEdgeActor(); - NKikimrSchemeBoard::TEvSubscribe::TCapabilities capabilities; - capabilities.SetAckNotifications(true); - Context->SubscribeReplica(Replica, subscriber, "path", true, 0, capabilities); - - // update to version 3 & omit ack - Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1), 3))); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); - UNIT_ASSERT_VALUES_EQUAL(3, ev->Get()->GetRecord().GetVersion()); - } - - // recreate path with version 1 - Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 2), 1))); - // ack previous version - Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(3)); - // should receive notification about recreated path - while (true) { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); - - if (ev->Get()->GetRecord().GetIsDeletion()) { - // deletion of previous version - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); - } else { - UNIT_ASSERT_VALUES_EQUAL(2, ev->Get()->GetRecord().GetLocalPathId()); - UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetVersion()); - break; - } - - Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(ev->Get()->GetRecord().GetVersion())); - } -} - -void TReplicaTest::StrongNotificationAfterCommit() { + + NKikimrSchemeBoard::TEvSubscribe::TCapabilities capabilities; + capabilities.SetAckNotifications(true); + Context->SubscribeReplica(Replica, subscriber, "path", true, 0, capabilities); + + Context->HandshakeReplica(Replica, populator); + + Context->Send(Replica, populator, GenerateUpdate(describe)); + Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(0)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); + + Context->Send(Replica, populator, GenerateUpdate(describe, 1, 1, true)); + Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(1)); + Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); +} + +void TReplicaTest::AckNotificationsUponPathRecreation() { const TActorId populator = Context->AllocateEdgeActor(); + Context->HandshakeReplica(Replica, populator); + + // initial path version is 2 + Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1), 2))); + const TActorId subscriber = Context->AllocateEdgeActor(); - - Context->SubscribeReplica(Replica, subscriber, "path", false, 1); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetStrong()); - } - - Context->HandshakeReplica(Replica, populator); - Context->CommitReplica(Replica, populator, 1, 1); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); - UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetStrong()); - } -} - + NKikimrSchemeBoard::TEvSubscribe::TCapabilities capabilities; + capabilities.SetAckNotifications(true); + Context->SubscribeReplica(Replica, subscriber, "path", true, 0, capabilities); + + // update to version 3 & omit ack + Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1), 3))); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); + UNIT_ASSERT_VALUES_EQUAL(3, ev->Get()->GetRecord().GetVersion()); + } + + // recreate path with version 1 + Context->Send(Replica, populator, GenerateUpdate(GenerateDescribe("path", TPathId(1, 2), 1))); + // ack previous version + Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(3)); + // should receive notification about recreated path + while (true) { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); + + if (ev->Get()->GetRecord().GetIsDeletion()) { + // deletion of previous version + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetLocalPathId()); + } else { + UNIT_ASSERT_VALUES_EQUAL(2, ev->Get()->GetRecord().GetLocalPathId()); + UNIT_ASSERT_VALUES_EQUAL(1, ev->Get()->GetRecord().GetVersion()); + break; + } + + Context->Send(Replica, subscriber, new TSchemeBoardEvents::TEvNotifyAck(ev->Get()->GetRecord().GetVersion())); + } +} + +void TReplicaTest::StrongNotificationAfterCommit() { + const TActorId populator = Context->AllocateEdgeActor(); + const TActorId subscriber = Context->AllocateEdgeActor(); + + Context->SubscribeReplica(Replica, subscriber, "path", false, 1); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->GetRecord().GetStrong()); + } + + Context->HandshakeReplica(Replica, populator); + Context->CommitReplica(Replica, populator, 1, 1); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(subscriber); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL(true, ev->Get()->GetRecord().GetStrong()); + } +} + class TReplicaCombinationTest: public NUnitTest::TTestBase { public: void SetUp() override { @@ -559,7 +559,7 @@ public: } TActorId GetReplica() { - return CreateReplica(*Context); + return CreateReplica(*Context); } void TearDown() override { @@ -671,14 +671,14 @@ void TReplicaCombinationTest::UpdatesCombinationsDomainRoot() { - UNIT_ASSERT_VALUES_EQUAL("/Root/Tenant", ev->Get()->GetRecord().GetPath()); - UNIT_ASSERT_VALUES_EQUAL(std::get<1>(winId), ev->Get()->GetRecord().GetIsDeletion()); + UNIT_ASSERT_VALUES_EQUAL("/Root/Tenant", ev->Get()->GetRecord().GetPath()); + UNIT_ASSERT_VALUES_EQUAL(std::get<1>(winId), ev->Get()->GetRecord().GetIsDeletion()); - if (!ev->Get()->GetRecord().GetIsDeletion()) { - UNIT_ASSERT_VALUES_EQUAL(std::get<2>(winId), TPathId(ev->Get()->GetRecord().GetPathOwnerId(), ev->Get()->GetRecord().GetLocalPathId())); - UNIT_ASSERT_VALUES_EQUAL(std::get<3>(winId), ev->Get()->GetRecord().GetVersion()); + if (!ev->Get()->GetRecord().GetIsDeletion()) { + UNIT_ASSERT_VALUES_EQUAL(std::get<2>(winId), TPathId(ev->Get()->GetRecord().GetPathOwnerId(), ev->Get()->GetRecord().GetLocalPathId())); + UNIT_ASSERT_VALUES_EQUAL(std::get<3>(winId), ev->Get()->GetRecord().GetVersion()); - UNIT_ASSERT(ev->Get()->GetRecord().HasDescribeSchemeResult()); + UNIT_ASSERT(ev->Get()->GetRecord().HasDescribeSchemeResult()); const NKikimrScheme::TEvDescribeSchemeResult& descr = ev->Get()->GetRecord().GetDescribeSchemeResult(); auto& domainKey = descr.GetPathDescription().GetDomainDescription().GetDomainKey(); UNIT_ASSERT_VALUES_EQUAL(std::get<0>(winId), TDomainId(domainKey.GetSchemeShard(), domainKey.GetPathId())); @@ -891,5 +891,5 @@ void TReplicaCombinationTest::UpdatesCombinationsMigratedPath() { } } -} // NSchemeBoard -} // NKikimr +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/scheme_board.h b/ydb/core/tx/scheme_board/scheme_board.h index f010ae5c341..fd329712488 100644 --- a/ydb/core/tx/scheme_board/scheme_board.h +++ b/ydb/core/tx/scheme_board/scheme_board.h @@ -1,7 +1,7 @@ -#pragma once - -#include "events.h" -#include "load_test.h" -#include "monitoring.h" -#include "replica.h" -#include "subscriber.h" +#pragma once + +#include "events.h" +#include "load_test.h" +#include "monitoring.h" +#include "replica.h" +#include "subscriber.h" diff --git a/ydb/core/tx/scheme_board/subscriber.cpp b/ydb/core/tx/scheme_board/subscriber.cpp index fd8c74f77e7..97173d81ad5 100644 --- a/ydb/core/tx/scheme_board/subscriber.cpp +++ b/ydb/core/tx/scheme_board/subscriber.cpp @@ -1,67 +1,67 @@ -#include "events.h" -#include "helpers.h" -#include "monitorable_actor.h" -#include "subscriber.h" - +#include "events.h" +#include "helpers.h" +#include "monitorable_actor.h" +#include "subscriber.h" + #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/base/tabletid.h> #include <ydb/core/protos/scheme_board.pb.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/util/yverify_stream.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/log.h> - -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> -#include <util/generic/set.h> -#include <util/generic/string.h> -#include <util/generic/utility.h> -#include <util/string/cast.h> - -namespace NKikimr { - -using EDeletionPolicy = ESchemeBoardSubscriberDeletionPolicy; - -namespace NSchemeBoard { - -#define SBS_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) -#define SBS_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) -#define SBS_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) -#define SBS_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) -#define SBS_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) -#define SBS_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) - -namespace { - - template <typename T, typename U> - typename std::enable_if_t<!std::is_same_v<T, U>, bool> IsSame(const T&, const U&) { - return false; - } - - template <typename T, typename U> - typename std::enable_if_t<std::is_same_v<T, U>, bool> IsSame(const T& t, const U& u) { - return t == u; - } - - template <typename TPath> - bool IsValidNotification(const TPath& path, const NKikimrSchemeBoard::TEvNotify& record) { - bool valid = false; - - if (record.HasPath()) { - valid = IsSame(path, record.GetPath()); - } - + +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> +#include <util/generic/set.h> +#include <util/generic/string.h> +#include <util/generic/utility.h> +#include <util/string/cast.h> + +namespace NKikimr { + +using EDeletionPolicy = ESchemeBoardSubscriberDeletionPolicy; + +namespace NSchemeBoard { + +#define SBS_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) +#define SBS_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) +#define SBS_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) +#define SBS_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) +#define SBS_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) +#define SBS_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "] " << this->SelfId() << " [" << Path << "] " << stream) + +namespace { + + template <typename T, typename U> + typename std::enable_if_t<!std::is_same_v<T, U>, bool> IsSame(const T&, const U&) { + return false; + } + + template <typename T, typename U> + typename std::enable_if_t<std::is_same_v<T, U>, bool> IsSame(const T& t, const U& u) { + return t == u; + } + + template <typename TPath> + bool IsValidNotification(const TPath& path, const NKikimrSchemeBoard::TEvNotify& record) { + bool valid = false; + + if (record.HasPath()) { + valid = IsSame(path, record.GetPath()); + } + if (!valid && (record.HasPathOwnerId() && record.HasLocalPathId())) { valid = IsSame(path, TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); - } - - return valid; - } - + } + + return valid; + } + struct TPathVersion { TPathId PathId; ui64 Version; @@ -72,21 +72,21 @@ namespace { { } - explicit TPathVersion(const TPathId& pathId, const ui64 version) + explicit TPathVersion(const TPathId& pathId, const ui64 version) : PathId(pathId) , Version(version) { } - static TPathVersion FromNotify(const NKikimrSchemeBoard::TEvNotify& record) { - TPathId pathId; - if (record.HasPathOwnerId() && record.HasLocalPathId()) { - pathId = TPathId(record.GetPathOwnerId(), record.GetLocalPathId()); - } - - return TPathVersion(pathId, record.GetVersion()); - } - + static TPathVersion FromNotify(const NKikimrSchemeBoard::TEvNotify& record) { + TPathId pathId; + if (record.HasPathOwnerId() && record.HasLocalPathId()) { + pathId = TPathId(record.GetPathOwnerId(), record.GetLocalPathId()); + } + + return TPathVersion(pathId, record.GetVersion()); + } + TString ToString() const { TString result; TStringOutput out(result); @@ -95,11 +95,11 @@ namespace { } void Out(IOutputStream& o) const { - if (!*this) { - PathId.Out(o); - } else { - o << "(PathId: " << PathId.ToString() << ", Version: " << Version << ")"; - } + if (!*this) { + PathId.Out(o); + } else { + o << "(PathId: " << PathId.ToString() << ", Version: " << Version << ")"; + } } bool operator<(const TPathVersion& x) const { @@ -125,94 +125,94 @@ namespace { } }; - struct TState { - bool Deleted = false; - bool Strong = false; - TPathVersion Version; - TDomainId DomainId; - TSet<ui64> AbandonedSchemeShards; - - TState() = default; - - private: - explicit TState(const TPathVersion& version, const TDomainId& domainId, const TSet<ui64>& abandonedSchemeShards) - : Deleted(false) - , Strong(true) - , Version(version) - , DomainId(domainId) - , AbandonedSchemeShards(abandonedSchemeShards) - { - } - - explicit TState(bool strong, const TPathVersion& version) - : Deleted(true) - , Strong(strong) - , Version(version) - { - } - - public: - static TState FromNotify(const NKikimrSchemeBoard::TEvNotify& record) { - if (!record.GetIsDeletion()) { - return TState( - TPathVersion::FromNotify(record), - GetDomainId(record.GetDescribeSchemeResult()), - GetAbandonedSchemeShardIds(record.GetDescribeSchemeResult()) - ); - } else { - return TState(record.GetStrong(), TPathVersion::FromNotify(record)); - } - } - - TString ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; - } - - void Out(IOutputStream& o) const { - o << "{" - << " Deleted: " << Deleted - << " Strong: " << Strong - << " Version: " << Version - << " DomainId: " << DomainId - << " AbandonedSchemeShards: " << "there are " << AbandonedSchemeShards.size() << " elements"; - if (AbandonedSchemeShards.size() > 0) { - o << ", first is " << *AbandonedSchemeShards.begin(); - } - if (AbandonedSchemeShards.size() > 1) { - o << ", last is " << *AbandonedSchemeShards.rbegin(); - } - o << " }"; - } - - bool LessThan(const TState& other, TString& reason) const { - if (!Strong && other.Strong) { - reason = "Update to strong state"; - return true; - } - - if (!other.Version) { - reason = "Ignore empty state"; - return false; - } - - if (!Version) { - reason = "Update to non-empty state"; - return true; - } - + struct TState { + bool Deleted = false; + bool Strong = false; + TPathVersion Version; + TDomainId DomainId; + TSet<ui64> AbandonedSchemeShards; + + TState() = default; + + private: + explicit TState(const TPathVersion& version, const TDomainId& domainId, const TSet<ui64>& abandonedSchemeShards) + : Deleted(false) + , Strong(true) + , Version(version) + , DomainId(domainId) + , AbandonedSchemeShards(abandonedSchemeShards) + { + } + + explicit TState(bool strong, const TPathVersion& version) + : Deleted(true) + , Strong(strong) + , Version(version) + { + } + + public: + static TState FromNotify(const NKikimrSchemeBoard::TEvNotify& record) { + if (!record.GetIsDeletion()) { + return TState( + TPathVersion::FromNotify(record), + GetDomainId(record.GetDescribeSchemeResult()), + GetAbandonedSchemeShardIds(record.GetDescribeSchemeResult()) + ); + } else { + return TState(record.GetStrong(), TPathVersion::FromNotify(record)); + } + } + + TString ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; + } + + void Out(IOutputStream& o) const { + o << "{" + << " Deleted: " << Deleted + << " Strong: " << Strong + << " Version: " << Version + << " DomainId: " << DomainId + << " AbandonedSchemeShards: " << "there are " << AbandonedSchemeShards.size() << " elements"; + if (AbandonedSchemeShards.size() > 0) { + o << ", first is " << *AbandonedSchemeShards.begin(); + } + if (AbandonedSchemeShards.size() > 1) { + o << ", last is " << *AbandonedSchemeShards.rbegin(); + } + o << " }"; + } + + bool LessThan(const TState& other, TString& reason) const { + if (!Strong && other.Strong) { + reason = "Update to strong state"; + return true; + } + + if (!other.Version) { + reason = "Ignore empty state"; + return false; + } + + if (!Version) { + reason = "Update to non-empty state"; + return true; + } + if (Version.PathId.OwnerId == other.Version.PathId.OwnerId) { - if (other.Version <= Version) { - reason = "Path was already updated"; - return false; - } - - reason = "Path was updated to new version"; - return true; - } - + if (other.Version <= Version) { + reason = "Path was already updated"; + return false; + } + + reason = "Path was updated to new version"; + return true; + } + if (!DomainId && Deleted) { if (other.Version <= Version) { reason = "Path was already deleted"; @@ -223,28 +223,28 @@ namespace { return true; } - // it is only because we need to manage undo of upgrade subdomain, finally remove it - - if (Version.PathId == other.DomainId) { //Update from TSS, GSS->TSS - if (AbandonedSchemeShards.contains(other.Version.PathId.OwnerId)) { //TSS is ignored, present GSS reverted that TSS - reason = "Update was ignored, GSS implisytly banned that TSS"; - return false; - } - - reason = "Path was updated as a replacement from TSS, GSS->TSS"; - return true; - } - - if (DomainId == other.Version.PathId) { //Update from GSS, TSS->GSS - if (other.AbandonedSchemeShards.contains(Version.PathId.OwnerId)) { //GSS reverts TSS - reason = "Path was updated as a replacement from GSS, GSS implicitly reverts TSS"; - return true; - } - - reason = "Update was ignored, TSS is preserved"; - return false; - } - + // it is only because we need to manage undo of upgrade subdomain, finally remove it + + if (Version.PathId == other.DomainId) { //Update from TSS, GSS->TSS + if (AbandonedSchemeShards.contains(other.Version.PathId.OwnerId)) { //TSS is ignored, present GSS reverted that TSS + reason = "Update was ignored, GSS implisytly banned that TSS"; + return false; + } + + reason = "Path was updated as a replacement from TSS, GSS->TSS"; + return true; + } + + if (DomainId == other.Version.PathId) { //Update from GSS, TSS->GSS + if (other.AbandonedSchemeShards.contains(Version.PathId.OwnerId)) { //GSS reverts TSS + reason = "Path was updated as a replacement from GSS, GSS implicitly reverts TSS"; + return true; + } + + reason = "Update was ignored, TSS is preserved"; + return false; + } + if (DomainId == other.DomainId) { if (other.Version <= Version) { reason = "Path was already updated"; @@ -261,799 +261,799 @@ namespace { return false; } - Y_FAIL_S("Unknown update" - << ": state# " << *this - << ", other state# " << other); - } - - bool LessThan(const TState& other) const { - TString unused; - return LessThan(other, unused); - } - - }; - -} // anonymous - -template <typename TPath, typename TDerived> -class TReplicaSubscriber: public TMonitorableActor<TDerived> { - void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { - auto& record = *ev->Get()->MutableRecord(); - - SBS_LOG_D("Handle TSchemeBoardEvents::TEvNotify" - << ": sender# " << ev->Sender); - SBS_LOG_T("Message:\n" << record.ShortDebugString()); - - this->Send(ev->Sender, new TSchemeBoardEvents::TEvNotifyAck(record.GetVersion())); - - if (!IsValidNotification(Path, record)) { - SBS_LOG_E("Suspicious notification" - << ": sender# " << ev->Sender - << ", other path# " << record.GetPath() - << ", other pathId# " << TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); - return; - } - - this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { - SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionRequest" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - - CurrentSyncRequest = ev->Cookie; - this->Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionResponse" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - - if (ev->Cookie != CurrentSyncRequest) { - return; - } - - this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); - CurrentSyncRequest = 0; - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); - auto& record = *response->Record.MutableReplicaSubscriberResponse(); - - ActorIdToProto(Parent, record.MutableParent()); - ActorIdToProto(Replica, record.MutableReplica()); - - record.SetDomainOwnerId(DomainOwnerId); - record.SetCurrentSyncRequest(CurrentSyncRequest); - - if constexpr (std::is_same_v<TPath, TString>) { - record.SetPath(Path); - } else { - record.MutablePathId()->SetOwnerId(Path.OwnerId); - record.MutablePathId()->SetLocalPathId(Path.LocalPathId); - } - - this->Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void PassAway() override { - if (Replica.NodeId() != this->SelfId().NodeId()) { - this->Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); - } - - this->Send(Replica, new TSchemeBoardEvents::TEvUnsubscribe(Path)); - this->Send(Parent, new TEvents::TEvGone()); - - TMonitorableActor<TDerived>::PassAway(); - } - - NJson::TJsonMap MonAttributes() const override { - return { - {"Parent", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, Parent)}, - {"Replica", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, - {"Path", ToString(Path)}, - }; - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { + Y_FAIL_S("Unknown update" + << ": state# " << *this + << ", other state# " << other); + } + + bool LessThan(const TState& other) const { + TString unused; + return LessThan(other, unused); + } + + }; + +} // anonymous + +template <typename TPath, typename TDerived> +class TReplicaSubscriber: public TMonitorableActor<TDerived> { + void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { + auto& record = *ev->Get()->MutableRecord(); + + SBS_LOG_D("Handle TSchemeBoardEvents::TEvNotify" + << ": sender# " << ev->Sender); + SBS_LOG_T("Message:\n" << record.ShortDebugString()); + + this->Send(ev->Sender, new TSchemeBoardEvents::TEvNotifyAck(record.GetVersion())); + + if (!IsValidNotification(Path, record)) { + SBS_LOG_E("Suspicious notification" + << ": sender# " << ev->Sender + << ", other path# " << record.GetPath() + << ", other pathId# " << TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); + return; + } + + this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { + SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionRequest" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + + CurrentSyncRequest = ev->Cookie; + this->Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { + SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionResponse" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + + if (ev->Cookie != CurrentSyncRequest) { + return; + } + + this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); + CurrentSyncRequest = 0; + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); + auto& record = *response->Record.MutableReplicaSubscriberResponse(); + + ActorIdToProto(Parent, record.MutableParent()); + ActorIdToProto(Replica, record.MutableReplica()); + + record.SetDomainOwnerId(DomainOwnerId); + record.SetCurrentSyncRequest(CurrentSyncRequest); + + if constexpr (std::is_same_v<TPath, TString>) { + record.SetPath(Path); + } else { + record.MutablePathId()->SetOwnerId(Path.OwnerId); + record.MutablePathId()->SetLocalPathId(Path.LocalPathId); + } + + this->Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void PassAway() override { + if (Replica.NodeId() != this->SelfId().NodeId()) { + this->Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); + } + + this->Send(Replica, new TSchemeBoardEvents::TEvUnsubscribe(Path)); + this->Send(Parent, new TEvents::TEvGone()); + + TMonitorableActor<TDerived>::PassAway(); + } + + NJson::TJsonMap MonAttributes() const override { + return { + {"Parent", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR, Parent)}, + {"Replica", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, + {"Path", ToString(Path)}, + }; + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { return "replica"sv; - } - - explicit TReplicaSubscriber( - const TActorId& parent, - const TActorId& replica, - const TPath& path, - const ui64 domainOwnerId) - : Parent(parent) - , Replica(replica) - , Path(path) - , DomainOwnerId(domainOwnerId) - , CurrentSyncRequest(0) - { - } - - void Bootstrap(const TActorContext&) { - TMonitorableActor<TDerived>::Bootstrap(); - - this->Send(Replica, new TSchemeBoardEvents::TEvSubscribe(Path, DomainOwnerId), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession); - this->Become(&TDerived::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvNotify, Handle); - hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); - hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, PassAway); - cFunc(TEvents::TEvUndelivered::EventType, PassAway); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - using TBase = TReplicaSubscriber<TPath, TDerived>; - -private: - const TActorId Parent; - const TActorId Replica; - const TPath Path; - const ui64 DomainOwnerId; - - ui64 CurrentSyncRequest; - -}; // TReplicaSubscriber - -class TReplicaSubscriberByPath: public TReplicaSubscriber<TString, TReplicaSubscriberByPath> { -public: - using TBase::TBase; -}; - -class TReplicaSubscriberByPathId: public TReplicaSubscriber<TPathId, TReplicaSubscriberByPathId> { -public: - using TBase::TBase; -}; - -template <typename TPath, typename TDerived, typename TReplicaDerived> -class TSubscriberProxy: public TMonitorableActor<TDerived> { - void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { - if (ev->Sender != ReplicaSubscriber) { - return; - } - - this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); - Delay = DefaultDelay; - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { + } + + explicit TReplicaSubscriber( + const TActorId& parent, + const TActorId& replica, + const TPath& path, + const ui64 domainOwnerId) + : Parent(parent) + , Replica(replica) + , Path(path) + , DomainOwnerId(domainOwnerId) + , CurrentSyncRequest(0) + { + } + + void Bootstrap(const TActorContext&) { + TMonitorableActor<TDerived>::Bootstrap(); + + this->Send(Replica, new TSchemeBoardEvents::TEvSubscribe(Path, DomainOwnerId), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession); + this->Become(&TDerived::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvNotify, Handle); + hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); + hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, PassAway); + cFunc(TEvents::TEvUndelivered::EventType, PassAway); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + using TBase = TReplicaSubscriber<TPath, TDerived>; + +private: + const TActorId Parent; + const TActorId Replica; + const TPath Path; + const ui64 DomainOwnerId; + + ui64 CurrentSyncRequest; + +}; // TReplicaSubscriber + +class TReplicaSubscriberByPath: public TReplicaSubscriber<TString, TReplicaSubscriberByPath> { +public: + using TBase::TBase; +}; + +class TReplicaSubscriberByPathId: public TReplicaSubscriber<TPathId, TReplicaSubscriberByPathId> { +public: + using TBase::TBase; +}; + +template <typename TPath, typename TDerived, typename TReplicaDerived> +class TSubscriberProxy: public TMonitorableActor<TDerived> { + void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { + if (ev->Sender != ReplicaSubscriber) { + return; + } + + this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); + Delay = DefaultDelay; + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { CurrentSyncRequest = ev->Cookie; - this->Send(ReplicaSubscriber, ev->Release().Release(), 0, ev->Cookie); - } - - void HandleSleep(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { - this->Send(Parent, new TSchemeBoardEvents::TEvSyncVersionResponse(0, true), 0, ev->Cookie); - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { + this->Send(ReplicaSubscriber, ev->Release().Release(), 0, ev->Cookie); + } + + void HandleSleep(TSchemeBoardEvents::TEvSyncVersionRequest::TPtr& ev) { + this->Send(Parent, new TSchemeBoardEvents::TEvSyncVersionResponse(0, true), 0, ev->Cookie); + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { if (ev->Sender != ReplicaSubscriber || ev->Cookie != CurrentSyncRequest) { - return; - } - - this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); + return; + } + + this->Send(Parent, ev->Release().Release(), 0, ev->Cookie); CurrentSyncRequest = 0; - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); - auto& record = *response->Record.MutableSubscriberProxyResponse(); - - ActorIdToProto(Parent, record.MutableParent()); - ActorIdToProto(Replica, record.MutableReplica()); - ActorIdToProto(ReplicaSubscriber, record.MutableReplicaSubscriber()); - - record.SetDomainOwnerId(DomainOwnerId); - record.SetCurrentSyncRequest(CurrentSyncRequest); - - if constexpr (std::is_same_v<TPath, TString>) { - record.SetPath(Path); - } else { - record.MutablePathId()->SetOwnerId(Path.OwnerId); - record.MutablePathId()->SetLocalPathId(Path.LocalPathId); - } - - this->Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void Handle(TEvents::TEvGone::TPtr& ev) { - if (ev->Sender != ReplicaSubscriber) { - return; - } - + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); + auto& record = *response->Record.MutableSubscriberProxyResponse(); + + ActorIdToProto(Parent, record.MutableParent()); + ActorIdToProto(Replica, record.MutableReplica()); + ActorIdToProto(ReplicaSubscriber, record.MutableReplicaSubscriber()); + + record.SetDomainOwnerId(DomainOwnerId); + record.SetCurrentSyncRequest(CurrentSyncRequest); + + if constexpr (std::is_same_v<TPath, TString>) { + record.SetPath(Path); + } else { + record.MutablePathId()->SetOwnerId(Path.OwnerId); + record.MutablePathId()->SetLocalPathId(Path.LocalPathId); + } + + this->Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void Handle(TEvents::TEvGone::TPtr& ev) { + if (ev->Sender != ReplicaSubscriber) { + return; + } + if (CurrentSyncRequest) { this->Send(Parent, new TSchemeBoardEvents::TEvSyncVersionResponse(0, true), 0, CurrentSyncRequest); CurrentSyncRequest = 0; } - ReplicaSubscriber = TActorId(); - this->Send(Parent, new TSchemeBoardEvents::TEvNotifyBuilder(Path, true)); - this->Become(&TDerived::StateSleep, Delay, new TEvents::TEvWakeup()); - Delay = Min(Delay * 2, MaxDelay); - } - - void PassAway() override { - if (ReplicaSubscriber) { - this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); - } - - TMonitorableActor<TDerived>::PassAway(); - } - - NJson::TJsonMap MonAttributes() const override { - return { - {"Parent", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, Parent)}, - {"Replica", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, - {"Path", ToString(Path)}, - }; - } - -public: + ReplicaSubscriber = TActorId(); + this->Send(Parent, new TSchemeBoardEvents::TEvNotifyBuilder(Path, true)); + this->Become(&TDerived::StateSleep, Delay, new TEvents::TEvWakeup()); + Delay = Min(Delay * 2, MaxDelay); + } + + void PassAway() override { + if (ReplicaSubscriber) { + this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); + } + + TMonitorableActor<TDerived>::PassAway(); + } + + NJson::TJsonMap MonAttributes() const override { + return { + {"Parent", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR, Parent)}, + {"Replica", TMonitorableActor<TDerived>::PrintActorIdAttr(NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_ACTOR, Replica)}, + {"Path", ToString(Path)}, + }; + } + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { + return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { return "proxy"sv; - } - - explicit TSubscriberProxy( - const TActorId& parent, - const TActorId& replica, - const TPath& path, - const ui64 domainOwnerId) - : Parent(parent) - , Replica(replica) - , Path(path) - , DomainOwnerId(domainOwnerId) - , Delay(DefaultDelay) + } + + explicit TSubscriberProxy( + const TActorId& parent, + const TActorId& replica, + const TPath& path, + const ui64 domainOwnerId) + : Parent(parent) + , Replica(replica) + , Path(path) + , DomainOwnerId(domainOwnerId) + , Delay(DefaultDelay) , CurrentSyncRequest(0) - { - } - - void Bootstrap(const TActorContext&) { - TMonitorableActor<TDerived>::Bootstrap(); - + { + } + + void Bootstrap(const TActorContext&) { + TMonitorableActor<TDerived>::Bootstrap(); + ReplicaSubscriber = this->RegisterWithSameMailbox(new TReplicaDerived(this->SelfId(), Replica, Path, DomainOwnerId)); - this->Become(&TDerived::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvNotify, Handle); - hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); - hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - hFunc(TEvents::TEvGone, Handle); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STFUNC(StateSleep) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, HandleSleep); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - CFunc(TEvents::TEvWakeup::EventType, Bootstrap); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - using TBase = TSubscriberProxy<TPath, TDerived, TReplicaDerived>; - -private: + this->Become(&TDerived::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvNotify, Handle); + hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, Handle); + hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + hFunc(TEvents::TEvGone, Handle); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STFUNC(StateSleep) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvSyncVersionRequest, HandleSleep); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + CFunc(TEvents::TEvWakeup::EventType, Bootstrap); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + using TBase = TSubscriberProxy<TPath, TDerived, TReplicaDerived>; + +private: const TActorId Parent; const TActorId Replica; - const TPath Path; - const ui64 DomainOwnerId; - - TActorId ReplicaSubscriber; - TDuration Delay; - + const TPath Path; + const ui64 DomainOwnerId; + + TActorId ReplicaSubscriber; + TDuration Delay; + ui64 CurrentSyncRequest; - static constexpr TDuration DefaultDelay = TDuration::MilliSeconds(10); - static constexpr TDuration MaxDelay = TDuration::Seconds(5); - -}; // TSubscriberProxy - -class TSubscriberProxyByPath: public TSubscriberProxy<TString, TSubscriberProxyByPath, TReplicaSubscriberByPath> { -public: - using TBase::TBase; -}; - -class TSubscriberProxyByPathId: public TSubscriberProxy<TPathId, TSubscriberProxyByPathId, TReplicaSubscriberByPathId> { -public: - using TBase::TBase; -}; - -template <typename TPath, typename TDerived, typename TProxyDerived> -class TSubscriber: public TMonitorableActor<TDerived> { - template <typename TNotify, typename... Args> - static THolder<TNotify> BuildNotify(const NKikimrSchemeBoard::TEvNotify& record, Args&&... args) { - THolder<TNotify> notify; - - TString path; - TPathId pathId; - - if (record.HasPath()) { - path = record.GetPath(); - } + static constexpr TDuration DefaultDelay = TDuration::MilliSeconds(10); + static constexpr TDuration MaxDelay = TDuration::Seconds(5); + +}; // TSubscriberProxy + +class TSubscriberProxyByPath: public TSubscriberProxy<TString, TSubscriberProxyByPath, TReplicaSubscriberByPath> { +public: + using TBase::TBase; +}; + +class TSubscriberProxyByPathId: public TSubscriberProxy<TPathId, TSubscriberProxyByPathId, TReplicaSubscriberByPathId> { +public: + using TBase::TBase; +}; + +template <typename TPath, typename TDerived, typename TProxyDerived> +class TSubscriber: public TMonitorableActor<TDerived> { + template <typename TNotify, typename... Args> + static THolder<TNotify> BuildNotify(const NKikimrSchemeBoard::TEvNotify& record, Args&&... args) { + THolder<TNotify> notify; + + TString path; + TPathId pathId; + + if (record.HasPath()) { + path = record.GetPath(); + } if (record.HasPathOwnerId() && record.HasLocalPathId()) { pathId = TPathId(record.GetPathOwnerId(), record.GetLocalPathId()); - } - - Y_VERIFY(path || pathId); - - if (!pathId) { + } + + Y_VERIFY(path || pathId); + + if (!pathId) { notify = MakeHolder<TNotify>(path, std::forward<Args>(args)...); - } else if (!path) { + } else if (!path) { notify = MakeHolder<TNotify>(pathId, std::forward<Args>(args)...); - } else { + } else { notify = MakeHolder<TNotify>(path, pathId, std::forward<Args>(args)...); - } - - return notify; - } - + } + + return notify; + } + TMap<TActorId, TState>::const_iterator SelectStateImpl() const { - Y_VERIFY(!States.empty()); - - auto it = States.begin(); - auto newest = it; - - while (++it != States.end()) { - if (newest->second.LessThan(it->second)) { - newest = it; - } - } - - return newest; - } - - const TState& SelectState() const { - auto it = SelectStateImpl(); - Y_VERIFY(it != States.end()); - return it->second; - } - - NKikimrSchemeBoard::TEvNotify& SelectResponse() { - Y_VERIFY(IsMajorityReached()); - - auto newest = SelectStateImpl(); - Y_VERIFY(newest != States.end()); - - auto it = InitialResponses.find(newest->first); - Y_VERIFY(it != InitialResponses.end()); - - return it->second; - } - - bool IsMajorityReached() const { - return InitialResponses.size() > (Proxies.size() / 2); - } - - void EnqueueSyncRequest(TSchemeBoardEvents::TEvSyncRequest::TPtr& ev) { - DelayedSyncRequest = Max(DelayedSyncRequest, ev->Cookie); - } - - bool MaybeRunVersionSync() { - if (!DelayedSyncRequest) { - return false; - } - - CurrentSyncRequest = DelayedSyncRequest; - DelayedSyncRequest = 0; - - Y_VERIFY(PendingSync.empty()); - for (const auto& proxy : Proxies) { - this->Send(proxy, new TSchemeBoardEvents::TEvSyncVersionRequest(Path), 0, CurrentSyncRequest); - PendingSync.emplace(proxy); - } - - return true; - } - - void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { - auto& record = *ev->Get()->MutableRecord(); - - SBS_LOG_D("Handle TSchemeBoardEvents::TEvNotify" - << ": sender# " << ev->Sender); - SBS_LOG_T("Message:\n" << record.ShortDebugString()); - - if (!IsValidNotification(Path, record)) { - SBS_LOG_E("Suspicious notification" - << ": sender# " << ev->Sender - << ", other path# " << record.GetPath() + Y_VERIFY(!States.empty()); + + auto it = States.begin(); + auto newest = it; + + while (++it != States.end()) { + if (newest->second.LessThan(it->second)) { + newest = it; + } + } + + return newest; + } + + const TState& SelectState() const { + auto it = SelectStateImpl(); + Y_VERIFY(it != States.end()); + return it->second; + } + + NKikimrSchemeBoard::TEvNotify& SelectResponse() { + Y_VERIFY(IsMajorityReached()); + + auto newest = SelectStateImpl(); + Y_VERIFY(newest != States.end()); + + auto it = InitialResponses.find(newest->first); + Y_VERIFY(it != InitialResponses.end()); + + return it->second; + } + + bool IsMajorityReached() const { + return InitialResponses.size() > (Proxies.size() / 2); + } + + void EnqueueSyncRequest(TSchemeBoardEvents::TEvSyncRequest::TPtr& ev) { + DelayedSyncRequest = Max(DelayedSyncRequest, ev->Cookie); + } + + bool MaybeRunVersionSync() { + if (!DelayedSyncRequest) { + return false; + } + + CurrentSyncRequest = DelayedSyncRequest; + DelayedSyncRequest = 0; + + Y_VERIFY(PendingSync.empty()); + for (const auto& proxy : Proxies) { + this->Send(proxy, new TSchemeBoardEvents::TEvSyncVersionRequest(Path), 0, CurrentSyncRequest); + PendingSync.emplace(proxy); + } + + return true; + } + + void Handle(TSchemeBoardEvents::TEvNotify::TPtr& ev) { + auto& record = *ev->Get()->MutableRecord(); + + SBS_LOG_D("Handle TSchemeBoardEvents::TEvNotify" + << ": sender# " << ev->Sender); + SBS_LOG_T("Message:\n" << record.ShortDebugString()); + + if (!IsValidNotification(Path, record)) { + SBS_LOG_E("Suspicious notification" + << ": sender# " << ev->Sender + << ", other path# " << record.GetPath() << ", other pathId# " << TPathId(record.GetPathOwnerId(), record.GetLocalPathId())); - return; - } - - States[ev->Sender] = TState::FromNotify(record); - if (!IsMajorityReached()) { - InitialResponses[ev->Sender] = std::move(record); - if (IsMajorityReached()) { - MaybeRunVersionSync(); - record = SelectResponse(); - } else { - return; - } - } - - const TState& newestState = SelectState(); - TString reason; - - if (!State) { - SBS_LOG_N("Set up state" - << ": owner# " << Owner - << ", state# " << newestState); - State = newestState; - } else if (State->LessThan(newestState, reason)) { - SBS_LOG_N("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); - State = newestState; - } else if (!State->Deleted && newestState.Deleted && newestState.Strong) { - SBS_LOG_N("Path was deleted" - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); - State = newestState; - } else { - SBS_LOG_I("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", other state# " << newestState); return; } - - if (!record.GetIsDeletion()) { + + States[ev->Sender] = TState::FromNotify(record); + if (!IsMajorityReached()) { + InitialResponses[ev->Sender] = std::move(record); + if (IsMajorityReached()) { + MaybeRunVersionSync(); + record = SelectResponse(); + } else { + return; + } + } + + const TState& newestState = SelectState(); + TString reason; + + if (!State) { + SBS_LOG_N("Set up state" + << ": owner# " << Owner + << ", state# " << newestState); + State = newestState; + } else if (State->LessThan(newestState, reason)) { + SBS_LOG_N("" << reason + << ": owner# " << Owner + << ", state# " << *State + << ", new state# " << newestState); + State = newestState; + } else if (!State->Deleted && newestState.Deleted && newestState.Strong) { + SBS_LOG_N("Path was deleted" + << ": owner# " << Owner + << ", state# " << *State + << ", new state# " << newestState); + State = newestState; + } else { + SBS_LOG_I("" << reason + << ": owner# " << Owner + << ", state# " << *State + << ", other state# " << newestState); + return; + } + + if (!record.GetIsDeletion()) { this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyUpdate>(record, std::move(*record.MutableDescribeSchemeResult()))); - } else { - this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyDelete>(record, record.GetStrong())); + } else { + this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyDelete>(record, record.GetStrong())); + } + } + + void Handle(TSchemeBoardEvents::TEvSyncRequest::TPtr& ev) { + SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncRequest" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + + if (ev->Sender != Owner) { + SBS_LOG_W("Suspicious sync version request" + << ": sender# " << ev->Sender + << ", owner# " << Owner); + return; + } + + EnqueueSyncRequest(ev); + + if (PendingSync || !IsMajorityReached()) { + return; + } + + Y_VERIFY(MaybeRunVersionSync()); + } + + void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { + SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionResponse" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + + if (ev->Cookie != CurrentSyncRequest) { + SBS_LOG_D("Sync cookie mismatch" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie + << ", current cookie# " << CurrentSyncRequest); + return; + } + + auto it = PendingSync.find(ev->Sender); + if (it == PendingSync.end()) { + SBS_LOG_D("Unexpected sync response" + << ": sender# " << ev->Sender + << ", cookie# " << ev->Cookie); + return; + } + + PendingSync.erase(it); + Y_VERIFY(!ReceivedSync.contains(ev->Sender)); + ReceivedSync[ev->Sender] = ev->Get()->Record.GetPartial(); + + ui32 successes = 0; + ui32 failures = 0; + for (const auto& [_, partial] : ReceivedSync) { + if (!partial) { + ++successes; + } else { + ++failures; + } + } + + const ui32 size = Proxies.size(); + const ui32 half = size / 2; + if (successes <= half && failures <= half && (successes + failures) < size) { + SBS_LOG_D("Sync is in progress" + << ": cookie# " << ev->Cookie + << ", size# " << size + << ", half# " << half + << ", successes# " << successes + << ", faulires# " << failures); + return; + } + + const bool partial = !(successes > half); + const TString done = TStringBuilder() << "Sync is done" + << ": cookie# " << ev->Cookie + << ", size# " << size + << ", half# " << half + << ", successes# " << successes + << ", faulires# " << failures + << ", partial# " << partial; + + if (!partial) { + SBS_LOG_D(done); + } else { + SBS_LOG_W(done); + } + + this->Send(Owner, new TSchemeBoardEvents::TEvSyncResponse(Path, partial), 0, ev->Cookie); + + PendingSync.clear(); + ReceivedSync.clear(); + + MaybeRunVersionSync(); + } + + void Handle(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) { + SBS_LOG_D("Handle TEvStateStorage::TEvResolveReplicasList"); + + const auto& replicas = ev->Get()->Replicas; + + if (replicas.empty()) { + SBS_LOG_E("Subscribe on unconfigured SchemeBoard" + << ": StateStorage group# " << StateStorageGroup); + this->Become(&TDerived::StateCalm); + return; } - } - - void Handle(TSchemeBoardEvents::TEvSyncRequest::TPtr& ev) { - SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncRequest" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - - if (ev->Sender != Owner) { - SBS_LOG_W("Suspicious sync version request" - << ": sender# " << ev->Sender - << ", owner# " << Owner); - return; - } - - EnqueueSyncRequest(ev); - - if (PendingSync || !IsMajorityReached()) { - return; - } - - Y_VERIFY(MaybeRunVersionSync()); - } - - void Handle(TSchemeBoardEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle TSchemeBoardEvents::TEvSyncVersionResponse" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - - if (ev->Cookie != CurrentSyncRequest) { - SBS_LOG_D("Sync cookie mismatch" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", current cookie# " << CurrentSyncRequest); - return; - } - - auto it = PendingSync.find(ev->Sender); - if (it == PendingSync.end()) { - SBS_LOG_D("Unexpected sync response" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); - return; - } - - PendingSync.erase(it); - Y_VERIFY(!ReceivedSync.contains(ev->Sender)); - ReceivedSync[ev->Sender] = ev->Get()->Record.GetPartial(); - - ui32 successes = 0; - ui32 failures = 0; - for (const auto& [_, partial] : ReceivedSync) { - if (!partial) { - ++successes; - } else { - ++failures; - } - } - - const ui32 size = Proxies.size(); - const ui32 half = size / 2; - if (successes <= half && failures <= half && (successes + failures) < size) { - SBS_LOG_D("Sync is in progress" - << ": cookie# " << ev->Cookie - << ", size# " << size - << ", half# " << half - << ", successes# " << successes - << ", faulires# " << failures); - return; - } - - const bool partial = !(successes > half); - const TString done = TStringBuilder() << "Sync is done" - << ": cookie# " << ev->Cookie - << ", size# " << size - << ", half# " << half - << ", successes# " << successes - << ", faulires# " << failures - << ", partial# " << partial; - - if (!partial) { - SBS_LOG_D(done); - } else { - SBS_LOG_W(done); - } - - this->Send(Owner, new TSchemeBoardEvents::TEvSyncResponse(Path, partial), 0, ev->Cookie); - - PendingSync.clear(); - ReceivedSync.clear(); - - MaybeRunVersionSync(); - } - - void Handle(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) { - SBS_LOG_D("Handle TEvStateStorage::TEvResolveReplicasList"); - - const auto& replicas = ev->Get()->Replicas; - - if (replicas.empty()) { - SBS_LOG_E("Subscribe on unconfigured SchemeBoard" - << ": StateStorage group# " << StateStorageGroup); - this->Become(&TDerived::StateCalm); - return; - } - - for (const auto& replica : replicas) { + + for (const auto& replica : replicas) { Proxies.emplace(this->RegisterWithSameMailbox(new TProxyDerived(this->SelfId(), replica, Path, DomainOwnerId))); - } - - this->Become(&TDerived::StateWork); - } - - void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { - auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); - auto& record = *response->Record.MutableSubscriberResponse(); - - ActorIdToProto(Owner, record.MutableOwner()); - - record.SetDomainOwnerId(DomainOwnerId); - record.SetDelayedSyncRequest(DelayedSyncRequest); - record.SetCurrentSyncRequest(CurrentSyncRequest); - - auto fillState = [](const auto& from, auto& to) { - to.SetDeleted(from.Deleted); - to.SetStrong(from.Strong); - to.SetVersion(from.Version.Version); - - to.MutablePathId()->SetOwnerId(from.Version.PathId.OwnerId); - to.MutablePathId()->SetLocalPathId(from.Version.PathId.LocalPathId); - - to.MutableDomainId()->SetOwnerId(from.DomainId.OwnerId); - to.MutableDomainId()->SetLocalPathId(from.DomainId.LocalPathId); - - for (const auto tabletId : from.AbandonedSchemeShards) { - to.AddAbandonedSchemeShards(tabletId); - } - }; - - for (const auto& [proxy, state] : States) { - auto& proxyState = *record.AddProxyStates(); - - ActorIdToProto(proxy, proxyState.MutableProxy()); - fillState(state, *proxyState.MutableState()); - } - - if (State) { - fillState(*State, *record.MutableState()); - } - - if constexpr (std::is_same_v<TPath, TString>) { - record.SetPath(Path); - } else { - record.MutablePathId()->SetOwnerId(Path.OwnerId); - record.MutablePathId()->SetLocalPathId(Path.LocalPathId); - } - - this->Send(ev->Sender, std::move(response), 0, ev->Cookie); - } - - void HandleUndelivered() { - SBS_LOG_E("Subscribe on unavailable SchemeBoard" - << ": StateStorage group# " << StateStorageGroup); - this->Become(&TDerived::StateCalm); - } - - void PassAway() override { - for (const auto& proxy : Proxies) { - this->Send(proxy, new TEvents::TEvPoisonPill()); - } - - TMonitorableActor<TDerived>::PassAway(); - } - - NJson::TJsonMap MonAttributes() const override { - return { - {"Owner", ToString(Owner)}, - {"Path", ToString(Path)}, - }; - } - -public: + } + + this->Become(&TDerived::StateWork); + } + + void Handle(TSchemeBoardMonEvents::TEvInfoRequest::TPtr& ev) { + auto response = MakeHolder<TSchemeBoardMonEvents::TEvInfoResponse>(this->SelfId(), this->ActorActivityType()); + auto& record = *response->Record.MutableSubscriberResponse(); + + ActorIdToProto(Owner, record.MutableOwner()); + + record.SetDomainOwnerId(DomainOwnerId); + record.SetDelayedSyncRequest(DelayedSyncRequest); + record.SetCurrentSyncRequest(CurrentSyncRequest); + + auto fillState = [](const auto& from, auto& to) { + to.SetDeleted(from.Deleted); + to.SetStrong(from.Strong); + to.SetVersion(from.Version.Version); + + to.MutablePathId()->SetOwnerId(from.Version.PathId.OwnerId); + to.MutablePathId()->SetLocalPathId(from.Version.PathId.LocalPathId); + + to.MutableDomainId()->SetOwnerId(from.DomainId.OwnerId); + to.MutableDomainId()->SetLocalPathId(from.DomainId.LocalPathId); + + for (const auto tabletId : from.AbandonedSchemeShards) { + to.AddAbandonedSchemeShards(tabletId); + } + }; + + for (const auto& [proxy, state] : States) { + auto& proxyState = *record.AddProxyStates(); + + ActorIdToProto(proxy, proxyState.MutableProxy()); + fillState(state, *proxyState.MutableState()); + } + + if (State) { + fillState(*State, *record.MutableState()); + } + + if constexpr (std::is_same_v<TPath, TString>) { + record.SetPath(Path); + } else { + record.MutablePathId()->SetOwnerId(Path.OwnerId); + record.MutablePathId()->SetLocalPathId(Path.LocalPathId); + } + + this->Send(ev->Sender, std::move(response), 0, ev->Cookie); + } + + void HandleUndelivered() { + SBS_LOG_E("Subscribe on unavailable SchemeBoard" + << ": StateStorage group# " << StateStorageGroup); + this->Become(&TDerived::StateCalm); + } + + void PassAway() override { + for (const auto& proxy : Proxies) { + this->Send(proxy, new TEvents::TEvPoisonPill()); + } + + TMonitorableActor<TDerived>::PassAway(); + } + + NJson::TJsonMap MonAttributes() const override { + return { + {"Owner", ToString(Owner)}, + {"Path", ToString(Path)}, + }; + } + +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { + } + + static constexpr TStringBuf LogPrefix() { return "main"sv; - } - - explicit TSubscriber( + } + + explicit TSubscriber( const TActorId& owner, - const TPath& path, - const ui64 stateStorageGroup, - const ui64 domainOwnerId) - : Owner(owner) - , Path(path) - , StateStorageGroup(stateStorageGroup) - , DomainOwnerId(domainOwnerId) - , DelayedSyncRequest(0) - , CurrentSyncRequest(0) - { - } - - void Bootstrap(const TActorContext&) { - TMonitorableActor<TDerived>::Bootstrap(); - + const TPath& path, + const ui64 stateStorageGroup, + const ui64 domainOwnerId) + : Owner(owner) + , Path(path) + , StateStorageGroup(stateStorageGroup) + , DomainOwnerId(domainOwnerId) + , DelayedSyncRequest(0) + , CurrentSyncRequest(0) + { + } + + void Bootstrap(const TActorContext&) { + TMonitorableActor<TDerived>::Bootstrap(); + const TActorId proxy = MakeStateStorageProxyID(StateStorageGroup); - this->Send(proxy, new TEvStateStorage::TEvResolveSchemeBoard(Path), IEventHandle::FlagTrackDelivery); - this->Become(&TDerived::StateResolve); - } - - STATEFN(StateResolve) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvSyncRequest, EnqueueSyncRequest); // from owner (cache) - - hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvents::TEvUndelivered::EventType, HandleUndelivered); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardEvents::TEvNotify, Handle); - hFunc(TSchemeBoardEvents::TEvSyncRequest, Handle); // from owner (cache) - hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); // from proxies - - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - STATEFN(StateCalm) { - switch (ev->GetTypeRewrite()) { - hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); - - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - - using TBase = TSubscriber<TPath, TDerived, TProxyDerived>; - -private: + this->Send(proxy, new TEvStateStorage::TEvResolveSchemeBoard(Path), IEventHandle::FlagTrackDelivery); + this->Become(&TDerived::StateResolve); + } + + STATEFN(StateResolve) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvSyncRequest, EnqueueSyncRequest); // from owner (cache) + + hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvents::TEvUndelivered::EventType, HandleUndelivered); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardEvents::TEvNotify, Handle); + hFunc(TSchemeBoardEvents::TEvSyncRequest, Handle); // from owner (cache) + hFunc(TSchemeBoardEvents::TEvSyncVersionResponse, Handle); // from proxies + + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + STATEFN(StateCalm) { + switch (ev->GetTypeRewrite()) { + hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); + + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + + using TBase = TSubscriber<TPath, TDerived, TProxyDerived>; + +private: const TActorId Owner; - const TPath Path; - const ui64 StateStorageGroup; - const ui64 DomainOwnerId; - - TSet<TActorId> Proxies; - TMap<TActorId, TState> States; + const TPath Path; + const ui64 StateStorageGroup; + const ui64 DomainOwnerId; + + TSet<TActorId> Proxies; + TMap<TActorId, TState> States; TMap<TActorId, NKikimrSchemeBoard::TEvNotify> InitialResponses; - TMaybe<TState> State; - - ui64 DelayedSyncRequest; - ui64 CurrentSyncRequest; - TSet<TActorId> PendingSync; - TMap<TActorId, bool> ReceivedSync; - -}; // TSubscriber - -class TSubscriberByPath: public TSubscriber<TString, TSubscriberByPath, TSubscriberProxyByPath> { -public: - using TBase::TBase; -}; - -class TSubscriberByPathId: public TSubscriber<TPathId, TSubscriberByPathId, TSubscriberProxyByPathId> { -public: - using TBase::TBase; -}; - -} // NSchemeBoard - -IActor* CreateSchemeBoardSubscriber( + TMaybe<TState> State; + + ui64 DelayedSyncRequest; + ui64 CurrentSyncRequest; + TSet<TActorId> PendingSync; + TMap<TActorId, bool> ReceivedSync; + +}; // TSubscriber + +class TSubscriberByPath: public TSubscriber<TString, TSubscriberByPath, TSubscriberProxyByPath> { +public: + using TBase::TBase; +}; + +class TSubscriberByPathId: public TSubscriber<TPathId, TSubscriberByPathId, TSubscriberProxyByPathId> { +public: + using TBase::TBase; +}; + +} // NSchemeBoard + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TString& path, - const ui64 stateStorageGroup, - const ui64 domainOwnerId -) { - return new NSchemeBoard::TSubscriberByPath(owner, path, stateStorageGroup, domainOwnerId); -} - -IActor* CreateSchemeBoardSubscriber( + const TString& path, + const ui64 stateStorageGroup, + const ui64 domainOwnerId +) { + return new NSchemeBoard::TSubscriberByPath(owner, path, stateStorageGroup, domainOwnerId); +} + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TPathId& pathId, - const ui64 stateStorageGroup, - const ui64 domainOwnerId -) { - return new NSchemeBoard::TSubscriberByPathId(owner, pathId, stateStorageGroup, domainOwnerId); -} - -IActor* CreateSchemeBoardSubscriber( + const TPathId& pathId, + const ui64 stateStorageGroup, + const ui64 domainOwnerId +) { + return new NSchemeBoard::TSubscriberByPathId(owner, pathId, stateStorageGroup, domainOwnerId); +} + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TString& path, - const ui64 stateStorageGroup, - const EDeletionPolicy deletionPolicy -) { - Y_UNUSED(deletionPolicy); - return new NSchemeBoard::TSubscriberByPath(owner, path, stateStorageGroup, 0); -} - -IActor* CreateSchemeBoardSubscriber( + const TString& path, + const ui64 stateStorageGroup, + const EDeletionPolicy deletionPolicy +) { + Y_UNUSED(deletionPolicy); + return new NSchemeBoard::TSubscriberByPath(owner, path, stateStorageGroup, 0); +} + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TPathId& pathId, - const ui64 stateStorageGroup, - const EDeletionPolicy deletionPolicy -) { - Y_UNUSED(deletionPolicy); - return new NSchemeBoard::TSubscriberByPathId(owner, pathId, stateStorageGroup, 0); -} - -} // NKikimr - -Y_DECLARE_OUT_SPEC(inline, NKikimr::NSchemeBoard::TPathVersion, o, x) { + const TPathId& pathId, + const ui64 stateStorageGroup, + const EDeletionPolicy deletionPolicy +) { + Y_UNUSED(deletionPolicy); + return new NSchemeBoard::TSubscriberByPathId(owner, pathId, stateStorageGroup, 0); +} + +} // NKikimr + +Y_DECLARE_OUT_SPEC(inline, NKikimr::NSchemeBoard::TPathVersion, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimr::NSchemeBoard::TState, o, x) { return x.Out(o); } - -Y_DECLARE_OUT_SPEC(inline, NKikimr::NSchemeBoard::TState, o, x) { - return x.Out(o); -} diff --git a/ydb/core/tx/scheme_board/subscriber.h b/ydb/core/tx/scheme_board/subscriber.h index 3d8fbb91d13..f415e4e9d57 100644 --- a/ydb/core/tx/scheme_board/subscriber.h +++ b/ydb/core/tx/scheme_board/subscriber.h @@ -1,46 +1,46 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/base/pathid.h> - -#include <util/generic/string.h> - -namespace NKikimr { - -IActor* CreateSchemeBoardSubscriber( + +#include <util/generic/string.h> + +namespace NKikimr { + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TString& path, - const ui64 stateStorageGroup, - const ui64 domainOwnerId -); - -IActor* CreateSchemeBoardSubscriber( + const TString& path, + const ui64 stateStorageGroup, + const ui64 domainOwnerId +); + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TPathId& pathId, - const ui64 stateStorageGroup, - const ui64 domainOwnerId -); - -// deprecated -enum class ESchemeBoardSubscriberDeletionPolicy { - First, - Second, - Majority, -}; - -IActor* CreateSchemeBoardSubscriber( + const TPathId& pathId, + const ui64 stateStorageGroup, + const ui64 domainOwnerId +); + +// deprecated +enum class ESchemeBoardSubscriberDeletionPolicy { + First, + Second, + Majority, +}; + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TString& path, - const ui64 stateStorageGroup, - const ESchemeBoardSubscriberDeletionPolicy deletionPolicy -); - -IActor* CreateSchemeBoardSubscriber( + const TString& path, + const ui64 stateStorageGroup, + const ESchemeBoardSubscriberDeletionPolicy deletionPolicy +); + +IActor* CreateSchemeBoardSubscriber( const TActorId& owner, - const TPathId& pathId, - const ui64 stateStorageGroup, - const ESchemeBoardSubscriberDeletionPolicy deletionPolicy -); - -} // NKikimr + const TPathId& pathId, + const ui64 stateStorageGroup, + const ESchemeBoardSubscriberDeletionPolicy deletionPolicy +); + +} // NKikimr diff --git a/ydb/core/tx/scheme_board/subscriber_ut.cpp b/ydb/core/tx/scheme_board/subscriber_ut.cpp index fdb7a9efa3b..7e1d580305f 100644 --- a/ydb/core/tx/scheme_board/subscriber_ut.cpp +++ b/ydb/core/tx/scheme_board/subscriber_ut.cpp @@ -1,254 +1,254 @@ -#include "events.h" -#include "subscriber.h" -#include "ut_helpers.h" - +#include "events.h" +#include "subscriber.h" +#include "ut_helpers.h" + #include <ydb/core/base/pathid.h> #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/testlib/basics/appdata.h> #include <ydb/core/testlib/basics/helpers.h> - + #include <library/cpp/actors/core/log.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/vector.h> -#include <util/generic/xrange.h> - -namespace NKikimr { -namespace NSchemeBoard { - -class TSubscriberTest: public NUnitTest::TTestBase { + +#include <util/generic/vector.h> +#include <util/generic/xrange.h> + +namespace NKikimr { +namespace NSchemeBoard { + +class TSubscriberTest: public NUnitTest::TTestBase { TVector<TActorId> ResolveReplicas() { const TActorId proxy = MakeStateStorageProxyID(0); const TActorId edge = Context->AllocateEdgeActor(); - - Context->Send(proxy, edge, new TEvStateStorage::TEvListSchemeBoard()); - auto ev = Context->GrabEdgeEvent<TEvStateStorage::TEvListSchemeBoardResult>(edge); - + + Context->Send(proxy, edge, new TEvStateStorage::TEvListSchemeBoard()); + auto ev = Context->GrabEdgeEvent<TEvStateStorage::TEvListSchemeBoardResult>(edge); + Y_VERIFY(ev->Get()->Info); auto allReplicas = ev->Get()->Info->SelectAllReplicas(); return TVector<TActorId>(allReplicas.begin(), allReplicas.end()); - } - -public: - void SetUp() override { - Context = MakeHolder<TTestContext>(2); - - for (ui32 i : xrange(Context->GetNodeCount())) { - SetupStateStorage(*Context, i, 0); - } - - Context->Initialize(TAppPrepare().Unwrap()); - Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_SUBSCRIBER, NLog::PRI_DEBUG); - } - - void TearDown() override { - Context.Reset(); - } - - UNIT_TEST_SUITE(TSubscriberTest); - UNIT_TEST(Boot); - UNIT_TEST(NotifyUpdate); - UNIT_TEST(NotifyDelete); - UNIT_TEST(StrongNotificationAfterCommit); - UNIT_TEST(InvalidNotification); - UNIT_TEST(ReconnectOnFailure); - UNIT_TEST(Sync); - UNIT_TEST(SyncPartial); - UNIT_TEST(SyncWithOutdatedReplica); - UNIT_TEST_SUITE_END(); - - void Boot(); - void NotifyUpdate(); - void NotifyDelete(); - void StrongNotificationAfterCommit(); - void InvalidNotification(); - void ReconnectOnFailure(); - void Sync(); - void SyncPartial(); - void SyncWithOutdatedReplica(); - -private: - THolder<TTestContext> Context; - -}; // TSubscriberTest - -UNIT_TEST_SUITE_REGISTRATION(TSubscriberTest); - -void TSubscriberTest::Boot() { + } + +public: + void SetUp() override { + Context = MakeHolder<TTestContext>(2); + + for (ui32 i : xrange(Context->GetNodeCount())) { + SetupStateStorage(*Context, i, 0); + } + + Context->Initialize(TAppPrepare().Unwrap()); + Context->SetLogPriority(NKikimrServices::SCHEME_BOARD_SUBSCRIBER, NLog::PRI_DEBUG); + } + + void TearDown() override { + Context.Reset(); + } + + UNIT_TEST_SUITE(TSubscriberTest); + UNIT_TEST(Boot); + UNIT_TEST(NotifyUpdate); + UNIT_TEST(NotifyDelete); + UNIT_TEST(StrongNotificationAfterCommit); + UNIT_TEST(InvalidNotification); + UNIT_TEST(ReconnectOnFailure); + UNIT_TEST(Sync); + UNIT_TEST(SyncPartial); + UNIT_TEST(SyncWithOutdatedReplica); + UNIT_TEST_SUITE_END(); + + void Boot(); + void NotifyUpdate(); + void NotifyDelete(); + void StrongNotificationAfterCommit(); + void InvalidNotification(); + void ReconnectOnFailure(); + void Sync(); + void SyncPartial(); + void SyncWithOutdatedReplica(); + +private: + THolder<TTestContext> Context; + +}; // TSubscriberTest + +UNIT_TEST_SUITE_REGISTRATION(TSubscriberTest); + +void TSubscriberTest::Boot() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->CreateSubscriber(edge, "path"); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); -} - -void TSubscriberTest::NotifyUpdate() { + + Context->CreateSubscriber(edge, "path"); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); +} + +void TSubscriberTest::NotifyUpdate() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path"); - - auto replicas = ResolveReplicas(); - - Context->HandshakeReplica(replicas[0], edge); + + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path"); + + auto replicas = ResolveReplicas(); + + Context->HandshakeReplica(replicas[0], edge); Context->Send(replicas[0], edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)))); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); -} - -void TSubscriberTest::NotifyDelete() { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); +} + +void TSubscriberTest::NotifyDelete() { const TActorId edge = Context->AllocateEdgeActor(); - - auto replicas = ResolveReplicas(); - Y_VERIFY(replicas.size() > 2); - - for (const auto& replica : replicas) { - Context->HandshakeReplica(replica, edge); + + auto replicas = ResolveReplicas(); + Y_VERIFY(replicas.size() > 2); + + for (const auto& replica : replicas) { + Context->HandshakeReplica(replica, edge); Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)))); - } - - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, "path"); - - for (size_t i = 0; i <= replicas.size() / 2 + 1; ++i) { - Context->Send(replicas[i], edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)), 1, 1, true)); - } - - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); -} - -void TSubscriberTest::StrongNotificationAfterCommit() { + } + + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, "path"); + + for (size_t i = 0; i <= replicas.size() / 2 + 1; ++i) { + Context->Send(replicas[i], edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)), 1, 1, true)); + } + + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); +} + +void TSubscriberTest::StrongNotificationAfterCommit() { const TActorId edge = Context->AllocateEdgeActor(); - - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path", 0, 1, false); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT(!ev->Get()->Strong); - } - - for (const auto& replica : ResolveReplicas()) { - Context->HandshakeReplica(replica, edge); - Context->CommitReplica(replica, edge); - } - - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT(ev->Get()->Strong); - } -} - -void TSubscriberTest::InvalidNotification() { + + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path", 0, 1, false); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT(!ev->Get()->Strong); + } + + for (const auto& replica : ResolveReplicas()) { + Context->HandshakeReplica(replica, edge); + Context->CommitReplica(replica, edge); + } + + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge); + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT(ev->Get()->Strong); + } +} + +void TSubscriberTest::InvalidNotification() { const TActorId edge = Context->AllocateEdgeActor(); - + const TActorId subscriber = Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path"); - - // send notification directly to subscriber - auto* notify = new TSchemeBoardEvents::TEvNotifyBuilder(TPathId(1, 1)); + + // send notification directly to subscriber + auto* notify = new TSchemeBoardEvents::TEvNotifyBuilder(TPathId(1, 1)); notify->Record.MutableDescribeSchemeResult()->CopyFrom(GenerateDescribe("another/path", TPathId(1, 1))); - Context->Send(subscriber, edge, notify); - - size_t counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotifyUpdate>(); - UNIT_ASSERT_VALUES_EQUAL(0, counter); -} - -void TSubscriberTest::ReconnectOnFailure() { + Context->Send(subscriber, edge, notify); + + size_t counter = Context->CountEdgeEvents<TSchemeBoardEvents::TEvNotifyUpdate>(); + UNIT_ASSERT_VALUES_EQUAL(0, counter); +} + +void TSubscriberTest::ReconnectOnFailure() { const TActorId edge = Context->AllocateEdgeActor(1); - - Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path", 0, 1, true, 1); - - Context->Disconnect(0, 1); - Context->Connect(0, 1); - - auto replicas = ResolveReplicas(); - - Context->HandshakeReplica(replicas[0], edge); + + Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path", 0, 1, true, 1); + + Context->Disconnect(0, 1); + Context->Connect(0, 1); + + auto replicas = ResolveReplicas(); + + Context->HandshakeReplica(replicas[0], edge); Context->Send(replicas[0], edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)))); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); -} - -void TSubscriberTest::Sync() { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); +} + +void TSubscriberTest::Sync() { const TActorId edge = Context->AllocateEdgeActor(); - - auto replicas = ResolveReplicas(); - for (const auto& replica : replicas) { - Context->HandshakeReplica(replica, edge); + + auto replicas = ResolveReplicas(); + for (const auto& replica : replicas) { + Context->HandshakeReplica(replica, edge); Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 1)))); - } - + } + const TActorId subscriber = Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyUpdate>(edge, "path"); - Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, 1); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->Partial); -} - -void TSubscriberTest::SyncPartial() { + Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, 1); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL(false, ev->Get()->Partial); +} + +void TSubscriberTest::SyncPartial() { const TActorId edge = Context->AllocateEdgeActor(); const TActorId subscriber = Context->CreateSubscriber<TSchemeBoardEvents::TEvNotifyDelete>(edge, "path"); - - ui64 syncCookie = 0; - auto replicas = ResolveReplicas(); - for (ui32 i : xrange(replicas.size())) { - Context->Send(replicas[i], edge, new TEvents::TEvPoisonPill()); - - Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, ++syncCookie); - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL((i + 1) > (replicas.size() / 2), ev->Get()->Partial); - } -} - -void TSubscriberTest::SyncWithOutdatedReplica() { - const TActorId edge = Context->AllocateEdgeActor(); - - auto replicas = ResolveReplicas(); - UNIT_ASSERT(replicas.size() > 2); - - for (ui32 i = 0; i < replicas.size(); ++i) { - const auto& replica = replicas.at(i); - Context->HandshakeReplica(replica, edge); - - if (i == 0) { - // outdated, but greater version - Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 2), 2, TDomainId(1, 1)))); - } else { - // up to date - Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(2, 2), 1, TDomainId(2, 1)))); - } - } - - const TActorId subscriber = Context->CreateSubscriber(edge, "path"); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); - UNIT_ASSERT_VALUES_EQUAL(TPathId(2, 2), ev->Get()->PathId); - } - - Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, 1); - { - auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); - UNIT_ASSERT(!ev->Get()->Partial); - } -} - + + ui64 syncCookie = 0; + auto replicas = ResolveReplicas(); + for (ui32 i : xrange(replicas.size())) { + Context->Send(replicas[i], edge, new TEvents::TEvPoisonPill()); + + Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, ++syncCookie); + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL((i + 1) > (replicas.size() / 2), ev->Get()->Partial); + } +} + +void TSubscriberTest::SyncWithOutdatedReplica() { + const TActorId edge = Context->AllocateEdgeActor(); + + auto replicas = ResolveReplicas(); + UNIT_ASSERT(replicas.size() > 2); + + for (ui32 i = 0; i < replicas.size(); ++i) { + const auto& replica = replicas.at(i); + Context->HandshakeReplica(replica, edge); + + if (i == 0) { + // outdated, but greater version + Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(1, 2), 2, TDomainId(1, 1)))); + } else { + // up to date + Context->Send(replica, edge, GenerateUpdate(GenerateDescribe("path", TPathId(2, 2), 1, TDomainId(2, 1)))); + } + } + + const TActorId subscriber = Context->CreateSubscriber(edge, "path"); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyUpdate>(edge); + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); + UNIT_ASSERT_VALUES_EQUAL(TPathId(2, 2), ev->Get()->PathId); + } + + Context->Send(subscriber, edge, new TSchemeBoardEvents::TEvSyncRequest(), 0, 1); + { + auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvSyncResponse>(edge); + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL("path", ev->Get()->Path); + UNIT_ASSERT(!ev->Get()->Partial); + } +} + class TSubscriberCombinationsTest: public NUnitTest::TTestBase { TVector<TActorId> ResolveReplicas(TTestContext& context) { const TActorId proxy = MakeStateStorageProxyID(0); @@ -508,5 +508,5 @@ void TSubscriberCombinationsTest::CombinationsMigratedPath() { } -} // NSchemeBoard -} // NKikimr +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/two_part_description.cpp b/ydb/core/tx/scheme_board/two_part_description.cpp index d6d96af0ef9..09d1c56f10d 100644 --- a/ydb/core/tx/scheme_board/two_part_description.cpp +++ b/ydb/core/tx/scheme_board/two_part_description.cpp @@ -1,15 +1,15 @@ -#include "two_part_description.h" - -namespace NKikimr { -namespace NSchemeBoard { - -bool TTwoPartDescription::Empty() const { - return PreSerialized.empty() && !Record.ByteSizeLong(); -} - -TTwoPartDescription::operator bool() const { - return !Empty(); -} - -} // NSchemeBoard -} // NKikimr +#include "two_part_description.h" + +namespace NKikimr { +namespace NSchemeBoard { + +bool TTwoPartDescription::Empty() const { + return PreSerialized.empty() && !Record.ByteSizeLong(); +} + +TTwoPartDescription::operator bool() const { + return !Empty(); +} + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/two_part_description.h b/ydb/core/tx/scheme_board/two_part_description.h index 90ddfca613c..0e7c8b48110 100644 --- a/ydb/core/tx/scheme_board/two_part_description.h +++ b/ydb/core/tx/scheme_board/two_part_description.h @@ -1,27 +1,27 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/flat_tx_scheme.pb.h> - -#include <util/generic/string.h> - -namespace NKikimr { -namespace NSchemeBoard { - -struct TTwoPartDescription { - TString PreSerialized; + +#include <util/generic/string.h> + +namespace NKikimr { +namespace NSchemeBoard { + +struct TTwoPartDescription { + TString PreSerialized; NKikimrScheme::TEvDescribeSchemeResult Record; - - TTwoPartDescription() = default; - + + TTwoPartDescription() = default; + explicit TTwoPartDescription(TString preSerialized, NKikimrScheme::TEvDescribeSchemeResult record) - : PreSerialized(std::move(preSerialized)) - , Record(std::move(record)) - { - } - - bool Empty() const; - operator bool() const; -}; - -} // NSchemeBoard -} // NKikimr + : PreSerialized(std::move(preSerialized)) + , Record(std::move(record)) + { + } + + bool Empty() const; + operator bool() const; +}; + +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/ut_cache/ya.make b/ydb/core/tx/scheme_board/ut_cache/ya.make index b539934ed9c..1b1d1200bcc 100644 --- a/ydb/core/tx/scheme_board/ut_cache/ya.make +++ b/ydb/core/tx/scheme_board/ut_cache/ya.make @@ -1,28 +1,28 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib ydb/core/tx/schemeshard ydb/core/tx/schemeshard/ut_helpers -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - cache_ut.cpp - ut_helpers.cpp -) - -END() +SRCS( + cache_ut.cpp + ut_helpers.cpp +) + +END() diff --git a/ydb/core/tx/scheme_board/ut_double_indexed/ya.make b/ydb/core/tx/scheme_board/ut_double_indexed/ya.make index 0eaae3ab3da..0ae0cf3659b 100644 --- a/ydb/core/tx/scheme_board/ut_double_indexed/ya.make +++ b/ydb/core/tx/scheme_board/ut_double_indexed/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( library/cpp/testing/unittest -) - -SRCS( - double_indexed_ut.cpp -) - -END() +) + +SRCS( + double_indexed_ut.cpp +) + +END() diff --git a/ydb/core/tx/scheme_board/ut_helpers.cpp b/ydb/core/tx/scheme_board/ut_helpers.cpp index c1474982f19..762d8ddea12 100644 --- a/ydb/core/tx/scheme_board/ut_helpers.cpp +++ b/ydb/core/tx/scheme_board/ut_helpers.cpp @@ -1,45 +1,45 @@ -#include "ut_helpers.h" - -namespace NKikimr { -namespace NSchemeBoard { - +#include "ut_helpers.h" + +namespace NKikimr { +namespace NSchemeBoard { + NKikimrScheme::TEvDescribeSchemeResult GenerateDescribe( - const TString& path, + const TString& path, TPathId pathId, ui64 version, TDomainId domainId -) { +) { NKikimrScheme::TEvDescribeSchemeResult describeSchemeResult; - - describeSchemeResult.SetPath(path); + + describeSchemeResult.SetPath(path); describeSchemeResult.SetPathId(pathId.LocalPathId); describeSchemeResult.SetPathOwnerId(pathId.OwnerId); - describeSchemeResult.MutablePathDescription()->MutableSelf()->SetPathVersion(version); - + describeSchemeResult.MutablePathDescription()->MutableSelf()->SetPathVersion(version); + if (domainId) { auto domainKey = describeSchemeResult.MutablePathDescription()->MutableDomainDescription()->MutableDomainKey(); domainKey->SetSchemeShard(domainId.OwnerId); domainKey->SetPathId(domainId.LocalPathId); } - return describeSchemeResult; -} - -TSchemeBoardEvents::TEvUpdate* GenerateUpdate( + return describeSchemeResult; +} + +TSchemeBoardEvents::TEvUpdate* GenerateUpdate( const NKikimrScheme::TEvDescribeSchemeResult& describe, - ui64 owner, - ui64 generation, - bool isDeletion -) { - auto* update = new TSchemeBoardEvents::TEvUpdateBuilder(owner, generation, describe, isDeletion); - - if (!isDeletion) { - update->SetDescribeSchemeResult(describe); - } - - return update; -} - + ui64 owner, + ui64 generation, + bool isDeletion +) { + auto* update = new TSchemeBoardEvents::TEvUpdateBuilder(owner, generation, describe, isDeletion); + + if (!isDeletion) { + update->SetDescribeSchemeResult(describe); + } + + return update; +} + TVector<TCombinationsArgs> GenerateCombinationsDomainRoot(TString path, ui64 gssOwnerID, TVector<ui64> tenantsOwners) { TVector<TCombinationsArgs> combinations; @@ -144,5 +144,5 @@ TVector<TCombinationsArgs> GenerateCombinationsMigratedPath(TString path, return combinations; } -} // NSchemeBoard -} // NKikimr +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/ut_helpers.h b/ydb/core/tx/scheme_board/ut_helpers.h index f0f7af31ac0..1f2e4d5e9b8 100644 --- a/ydb/core/tx/scheme_board/ut_helpers.h +++ b/ydb/core/tx/scheme_board/ut_helpers.h @@ -1,9 +1,9 @@ -#pragma once - -#include "defs.h" -#include "events.h" -#include "subscriber.h" - +#pragma once + +#include "defs.h" +#include "events.h" +#include "subscriber.h" + #include <ydb/core/base/tablet_types.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/testlib/basics/appdata.h> @@ -13,246 +13,246 @@ #include <ydb/core/testlib/tablet_helpers.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/tx_allocator/txallocator.h> - + #include <library/cpp/actors/interconnect/interconnect_impl.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/algorithm.h> -#include <util/generic/string.h> - -namespace NKikimr { -namespace NSchemeBoard { - -class TTestContext: public TTestBasicRuntime { -public: - using TTestBasicRuntime::TTestBasicRuntime; - - void Send( + +#include <util/generic/algorithm.h> +#include <util/generic/string.h> + +namespace NKikimr { +namespace NSchemeBoard { + +class TTestContext: public TTestBasicRuntime { +public: + using TTestBasicRuntime::TTestBasicRuntime; + + void Send( const TActorId& recipient, const TActorId& sender, - IEventBase* ev, - ui32 flags = 0, - ui64 cookie = 0, - ui32 senderNodeIndex = 0, - bool viaActorSystem = false - ) { - TTestBasicRuntime::Send(new IEventHandle(recipient, sender, ev, flags, cookie), senderNodeIndex, viaActorSystem); - } - - void WaitForEvent(ui32 eventType) { - TDispatchOptions options; - options.FinalEvents.emplace_back(eventType); - DispatchEvents(options); - } - - void Connect(ui32 nodeIndexFrom, ui32 nodeIndexTo) { + IEventBase* ev, + ui32 flags = 0, + ui64 cookie = 0, + ui32 senderNodeIndex = 0, + bool viaActorSystem = false + ) { + TTestBasicRuntime::Send(new IEventHandle(recipient, sender, ev, flags, cookie), senderNodeIndex, viaActorSystem); + } + + void WaitForEvent(ui32 eventType) { + TDispatchOptions options; + options.FinalEvents.emplace_back(eventType); + DispatchEvents(options); + } + + void Connect(ui32 nodeIndexFrom, ui32 nodeIndexTo) { const TActorId proxy = GetInterconnectProxy(nodeIndexFrom, nodeIndexTo); - + Send(proxy, TActorId(), new TEvInterconnect::TEvConnectNode(), 0, 0, nodeIndexFrom, true); - WaitForEvent(TEvInterconnect::EvNodeConnected); - } - - void Disconnect(ui32 nodeIndexFrom, ui32 nodeIndexTo) { + WaitForEvent(TEvInterconnect::EvNodeConnected); + } + + void Disconnect(ui32 nodeIndexFrom, ui32 nodeIndexTo) { const TActorId proxy = GetInterconnectProxy(nodeIndexFrom, nodeIndexTo); - + Send(proxy, TActorId(), new TEvInterconnect::TEvDisconnect(), 0, 0, nodeIndexFrom, true); - WaitForEvent(TEvInterconnect::EvNodeDisconnected); - } - - template <typename TEvent> - size_t CountEvents(bool dispatch = true) { - if (dispatch) { - if (!DispatchEvents()) { - return 0; - } - } - - return CountIf(CaptureEvents(), [](const TAutoPtr<IEventHandle> ev) { - return ev->GetTypeRewrite() == TEvent::EventType; - }); - } - - template <typename TEvent> - size_t CountEdgeEvents() { - return CountEvents<TEvent>(false); - } - - TSchemeBoardEvents::TEvHandshakeResponse::TPtr HandshakeReplica( + WaitForEvent(TEvInterconnect::EvNodeDisconnected); + } + + template <typename TEvent> + size_t CountEvents(bool dispatch = true) { + if (dispatch) { + if (!DispatchEvents()) { + return 0; + } + } + + return CountIf(CaptureEvents(), [](const TAutoPtr<IEventHandle> ev) { + return ev->GetTypeRewrite() == TEvent::EventType; + }); + } + + template <typename TEvent> + size_t CountEdgeEvents() { + return CountEvents<TEvent>(false); + } + + TSchemeBoardEvents::TEvHandshakeResponse::TPtr HandshakeReplica( const TActorId& replica, const TActorId& sender, - ui64 owner = 1, - ui64 generation = 1, - bool grabResponse = true - ) { - Send(replica, sender, new TSchemeBoardEvents::TEvHandshakeRequest(owner, generation)); - - if (grabResponse) { - return GrabEdgeEvent<TSchemeBoardEvents::TEvHandshakeResponse>(sender); - } - - return nullptr; - } - - void CommitReplica( + ui64 owner = 1, + ui64 generation = 1, + bool grabResponse = true + ) { + Send(replica, sender, new TSchemeBoardEvents::TEvHandshakeRequest(owner, generation)); + + if (grabResponse) { + return GrabEdgeEvent<TSchemeBoardEvents::TEvHandshakeResponse>(sender); + } + + return nullptr; + } + + void CommitReplica( const TActorId& replica, const TActorId& sender, - ui64 owner = 1, - ui64 generation = 1 - ) { - Send(replica, sender, new TSchemeBoardEvents::TEvCommitRequest(owner, generation)); - } - - template <typename TPath> - TSchemeBoardEvents::TEvNotify::TPtr SubscribeReplica( + ui64 owner = 1, + ui64 generation = 1 + ) { + Send(replica, sender, new TSchemeBoardEvents::TEvCommitRequest(owner, generation)); + } + + template <typename TPath> + TSchemeBoardEvents::TEvNotify::TPtr SubscribeReplica( const TActorId& replica, const TActorId& sender, - const TPath& path, - bool grabResponse = true, - const ui64 domainOwnerId = 0, - const NKikimrSchemeBoard::TEvSubscribe::TCapabilities& capabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities() - ) { - auto subscribe = MakeHolder<TSchemeBoardEvents::TEvSubscribe>(path, domainOwnerId); - subscribe->Record.MutableCapabilities()->CopyFrom(capabilities); - - Send(replica, sender, subscribe.Release()); - - if (grabResponse) { - return GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(sender); - } - - return nullptr; - } - - template <typename TPath> + const TPath& path, + bool grabResponse = true, + const ui64 domainOwnerId = 0, + const NKikimrSchemeBoard::TEvSubscribe::TCapabilities& capabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities() + ) { + auto subscribe = MakeHolder<TSchemeBoardEvents::TEvSubscribe>(path, domainOwnerId); + subscribe->Record.MutableCapabilities()->CopyFrom(capabilities); + + Send(replica, sender, subscribe.Release()); + + if (grabResponse) { + return GrabEdgeEvent<TSchemeBoardEvents::TEvNotify>(sender); + } + + return nullptr; + } + + template <typename TPath> void UnsubscribeReplica(const TActorId& replica, const TActorId& sender, const TPath& path) { - Send(replica, sender, new TSchemeBoardEvents::TEvUnsubscribe(path)); - } - - template <typename TEvent, typename TPath> + Send(replica, sender, new TSchemeBoardEvents::TEvUnsubscribe(path)); + } + + template <typename TEvent, typename TPath> TActorId CreateSubscriber( const TActorId& owner, - const TPath& path, - ui64 stateStorageGroup = 0, - ui64 domainOwnerId = 1, - bool grabResponse = true, - ui32 nodeIndex = 0 - ) { + const TPath& path, + ui64 stateStorageGroup = 0, + ui64 domainOwnerId = 1, + bool grabResponse = true, + ui32 nodeIndex = 0 + ) { const TActorId subscriber = Register( - CreateSchemeBoardSubscriber(owner, path, stateStorageGroup, domainOwnerId), nodeIndex - ); - EnableScheduleForActor(subscriber, true); - - if (grabResponse) { - GrabEdgeEvent<TEvent>(owner); - } - - return subscriber; - } - - template <typename TPath> + CreateSchemeBoardSubscriber(owner, path, stateStorageGroup, domainOwnerId), nodeIndex + ); + EnableScheduleForActor(subscriber, true); + + if (grabResponse) { + GrabEdgeEvent<TEvent>(owner); + } + + return subscriber; + } + + template <typename TPath> TActorId CreateSubscriber( const TActorId& owner, - const TPath& path, - ui64 stateStorageGroup = 0, - ui64 domainOwnerId = 1, - ui32 nodeIndex = 0 - ) { - return CreateSubscriber<TSchemeBoardEvents::TEvNotify>( - owner, path, stateStorageGroup, domainOwnerId, false, nodeIndex - ); - } - -}; // TTestContext - -class TTestWithSchemeshard: public NUnitTest::TTestBase { - static void AddDomain( - TTestActorRuntime& runtime, - TAppPrepare& app, - const TString& name, - ui32 domainUid, - ui32 stateStorageGroup, - ui64 hiveTabletId, - ui64 schemeshardTabletId - ) { - app.ClearDomainsAndHive(); - ui32 planResolution = 50; - auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds( - name, domainUid, schemeshardTabletId, + const TPath& path, + ui64 stateStorageGroup = 0, + ui64 domainOwnerId = 1, + ui32 nodeIndex = 0 + ) { + return CreateSubscriber<TSchemeBoardEvents::TEvNotify>( + owner, path, stateStorageGroup, domainOwnerId, false, nodeIndex + ); + } + +}; // TTestContext + +class TTestWithSchemeshard: public NUnitTest::TTestBase { + static void AddDomain( + TTestActorRuntime& runtime, + TAppPrepare& app, + const TString& name, + ui32 domainUid, + ui32 stateStorageGroup, + ui64 hiveTabletId, + ui64 schemeshardTabletId + ) { + app.ClearDomainsAndHive(); + ui32 planResolution = 50; + auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds( + name, domainUid, schemeshardTabletId, stateStorageGroup, stateStorageGroup, TVector<ui32>{stateStorageGroup}, - domainUid, TVector<ui32>{domainUid}, - planResolution, - TVector<ui64>{TDomainsInfo::MakeTxCoordinatorIDFixed(domainUid, 1)}, - TVector<ui64>{}, - TVector<ui64>{TDomainsInfo::MakeTxAllocatorIDFixed(domainUid, 1)}, - DefaultPoolKinds(2) - ); - - TVector<ui64> ids = runtime.GetTxAllocatorTabletIds(); - ids.insert(ids.end(), domain->TxAllocators.begin(), domain->TxAllocators.end()); - runtime.SetTxAllocatorTabletIds(ids); - - app.AddDomain(domain.Release()); - app.AddHive(domainUid, hiveTabletId); - } - - static void SetupRuntime(TTestActorRuntime& runtime) { - for (ui32 i : xrange(runtime.GetNodeCount())) { - SetupStateStorage(runtime, i, 0); - } - - TAppPrepare app; + domainUid, TVector<ui32>{domainUid}, + planResolution, + TVector<ui64>{TDomainsInfo::MakeTxCoordinatorIDFixed(domainUid, 1)}, + TVector<ui64>{}, + TVector<ui64>{TDomainsInfo::MakeTxAllocatorIDFixed(domainUid, 1)}, + DefaultPoolKinds(2) + ); + + TVector<ui64> ids = runtime.GetTxAllocatorTabletIds(); + ids.insert(ids.end(), domain->TxAllocators.begin(), domain->TxAllocators.end()); + runtime.SetTxAllocatorTabletIds(ids); + + app.AddDomain(domain.Release()); + app.AddHive(domainUid, hiveTabletId); + } + + static void SetupRuntime(TTestActorRuntime& runtime) { + for (ui32 i : xrange(runtime.GetNodeCount())) { + SetupStateStorage(runtime, i, 0); + } + + TAppPrepare app; AddDomain(runtime, app, "Root", 0, 0, TTestTxConfig::Hive, TTestTxConfig::SchemeShard); SetupChannelProfiles(app, 0, 1); - SetupTabletServices(runtime, &app, true); - } - - static void BootSchemeShard(TTestActorRuntime& runtime, ui64 tabletId) { + SetupTabletServices(runtime, &app, true); + } + + static void BootSchemeShard(TTestActorRuntime& runtime, ui64 tabletId) { using namespace NSchemeShard; - + CreateTestBootstrapper(runtime, CreateTestTabletInfo(tabletId, TTabletTypes::FLAT_SCHEMESHARD), &CreateFlatTxSchemeShard); - + const TActorId edge = runtime.AllocateEdgeActor(); - + auto init = new TEvSchemeShard::TEvInitRootShard(edge, 32, "Root"); - runtime.SendToPipe(tabletId, edge, init, 0, GetPipeConfigWithRetries()); + runtime.SendToPipe(tabletId, edge, init, 0, GetPipeConfigWithRetries()); auto ev = runtime.GrabEdgeEvent<TEvSchemeShard::TEvInitRootShardResult>(edge); - - UNIT_ASSERT(ev->Get()); - UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetOrigin(), tabletId); + + UNIT_ASSERT(ev->Get()); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetOrigin(), tabletId); UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), (ui32)TEvSchemeShard::TEvInitRootShardResult::StatusAlreadyInitialized); - } - - static void BootTxAllocator(TTestActorRuntime& runtime, ui64 tabletId) { + } + + static void BootTxAllocator(TTestActorRuntime& runtime, ui64 tabletId) { CreateTestBootstrapper(runtime, CreateTestTabletInfo(tabletId, TTabletTypes::TX_ALLOCATOR), &CreateTxAllocator); - } - - void BootCoordinator(TTestActorRuntime& runtime, ui64 tabletId) { - CoordinatorState = new TFakeCoordinator::TState(); + } + + void BootCoordinator(TTestActorRuntime& runtime, ui64 tabletId) { + CoordinatorState = new TFakeCoordinator::TState(); BootFakeCoordinator(runtime, tabletId, CoordinatorState); - } - - void BootHive(TTestActorRuntime& runtime, ui64 tabletId) { - HiveState = new TFakeHiveState(); - BootFakeHive(runtime, tabletId, HiveState); - } - -protected: - virtual TTestContext::TEventObserver ObserverFunc() { - return TTestContext::DefaultObserverFunc; - } - -public: - void SetUp() override { - Context = MakeHolder<TTestContext>(); - Context->SetObserverFunc(ObserverFunc()); - - SetupRuntime(*Context); + } + + void BootHive(TTestActorRuntime& runtime, ui64 tabletId) { + HiveState = new TFakeHiveState(); + BootFakeHive(runtime, tabletId, HiveState); + } + +protected: + virtual TTestContext::TEventObserver ObserverFunc() { + return TTestContext::DefaultObserverFunc; + } + +public: + void SetUp() override { + Context = MakeHolder<TTestContext>(); + Context->SetObserverFunc(ObserverFunc()); + + SetupRuntime(*Context); BootSchemeShard(*Context, TTestTxConfig::SchemeShard); BootTxAllocator(*Context, TTestTxConfig::TxAllocator); BootCoordinator(*Context, TTestTxConfig::Coordinator); BootHive(*Context, TTestTxConfig::Hive); - } - + } + void TurnOnTabletsScheduling() { if (SchedulingGuard) { return; @@ -272,37 +272,37 @@ public: RebootTablet(*Context, (ui64)TTestTxConfig::SchemeShard, sender); } - void TearDown() override { + void TearDown() override { SchedulingGuard.Reset(); - CoordinatorState.Drop(); - HiveState.Drop(); + CoordinatorState.Drop(); + HiveState.Drop(); Context.Reset(); - } - -protected: - THolder<TTestContext> Context; - -private: - TFakeCoordinator::TState::TPtr CoordinatorState; - TFakeHiveState::TPtr HiveState; + } + +protected: + THolder<TTestContext> Context; + +private: + TFakeCoordinator::TState::TPtr CoordinatorState; + TFakeHiveState::TPtr HiveState; THolder<ITabletScheduledEventsGuard> SchedulingGuard; - -}; // TTestWithSchemeshard - + +}; // TTestWithSchemeshard + NKikimrScheme::TEvDescribeSchemeResult GenerateDescribe( - const TString& path, + const TString& path, TPathId pathId, ui64 version = 1, TDomainId domainId = TDomainId() -); - -TSchemeBoardEvents::TEvUpdate* GenerateUpdate( +); + +TSchemeBoardEvents::TEvUpdate* GenerateUpdate( const NKikimrScheme::TEvDescribeSchemeResult& describe, - ui64 owner = 1, - ui64 generation = 1, - bool isDeletion = false -); - + ui64 owner = 1, + ui64 generation = 1, + bool isDeletion = false +); + struct TCombinationsArgs { TString Path; TPathId PathId; @@ -333,5 +333,5 @@ TVector<TCombinationsArgs> GenerateCombinationsMigratedPath(TString path, ui64 gssID, TVector<ui64> tssIDs, ui64 gssLocalPathId, ui64 tssLocalPathId); -} // NSchemeBoard -} // NKikimr +} // NSchemeBoard +} // NKikimr diff --git a/ydb/core/tx/scheme_board/ut_monitoring/ya.make b/ydb/core/tx/scheme_board/ut_monitoring/ya.make index 66410f486dd..86a1deb5ed3 100644 --- a/ydb/core/tx/scheme_board/ut_monitoring/ya.make +++ b/ydb/core/tx/scheme_board/ut_monitoring/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( - library/cpp/actors/core - library/cpp/testing/unittest + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( + library/cpp/actors/core + library/cpp/testing/unittest ydb/core/testlib/basics -) - +) + YQL_LAST_ABI_VERSION() - -SRCS( - monitoring_ut.cpp - ut_helpers.cpp -) - -END() + +SRCS( + monitoring_ut.cpp + ut_helpers.cpp +) + +END() diff --git a/ydb/core/tx/scheme_board/ut_populator/ya.make b/ydb/core/tx/scheme_board/ut_populator/ya.make index fa1249ed95a..c5bf28ef045 100644 --- a/ydb/core/tx/scheme_board/ut_populator/ya.make +++ b/ydb/core/tx/scheme_board/ut_populator/ya.make @@ -1,29 +1,29 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() SIZE(MEDIUM) TIMEOUT(600) - -PEERDIR( + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib ydb/core/tx/schemeshard ydb/core/tx/schemeshard/ut_helpers ydb/core/tx/tx_allocator -) - -SRCS( - populator_ut.cpp - ut_helpers.cpp -) - +) + +SRCS( + populator_ut.cpp + ut_helpers.cpp +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/tx/scheme_board/ut_replica/ya.make b/ydb/core/tx/scheme_board/ut_replica/ya.make index 1b21d5cc0c7..690a11fbe75 100644 --- a/ydb/core/tx/scheme_board/ut_replica/ya.make +++ b/ydb/core/tx/scheme_board/ut_replica/ya.make @@ -1,26 +1,26 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib/basics -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - replica_ut.cpp - ut_helpers.cpp -) - -END() +SRCS( + replica_ut.cpp + ut_helpers.cpp +) + +END() diff --git a/ydb/core/tx/scheme_board/ut_subscriber/ya.make b/ydb/core/tx/scheme_board/ut_subscriber/ya.make index 824ad70f015..cf3dd5d1d01 100644 --- a/ydb/core/tx/scheme_board/ut_subscriber/ya.make +++ b/ydb/core/tx/scheme_board/ut_subscriber/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/core/tx/scheme_board) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SIZE(MEDIUM) - -TIMEOUT(600) - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( library/cpp/actors/interconnect library/cpp/testing/unittest ydb/core/testlib/basics -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - subscriber_ut.cpp - ut_helpers.cpp -) - -END() +SRCS( + subscriber_ut.cpp + ut_helpers.cpp +) + +END() diff --git a/ydb/core/tx/scheme_board/ya.make b/ydb/core/tx/scheme_board/ya.make index e319825dff5..3caed17371a 100644 --- a/ydb/core/tx/scheme_board/ya.make +++ b/ydb/core/tx/scheme_board/ya.make @@ -1,11 +1,11 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( library/cpp/actors/core ydb/core/base ydb/core/mon @@ -14,22 +14,22 @@ PEERDIR( ydb/core/tx/scheme_cache ydb/core/util ydb/library/aclib -) - -SRCS( - cache.cpp - helpers.cpp - load_test.cpp - monitoring.cpp - populator.cpp - replica.cpp - subscriber.cpp - two_part_description.cpp -) - -GENERATE_ENUM_SERIALIZATION(subscriber.h) - -END() +) + +SRCS( + cache.cpp + helpers.cpp + load_test.cpp + monitoring.cpp + populator.cpp + replica.cpp + subscriber.cpp + two_part_description.cpp +) + +GENERATE_ENUM_SERIALIZATION(subscriber.h) + +END() RECURSE_FOR_TESTS( ut_cache diff --git a/ydb/core/tx/scheme_cache/scheme_cache.cpp b/ydb/core/tx/scheme_cache/scheme_cache.cpp index cb190919587..5cf9a0b0d5e 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.cpp +++ b/ydb/core/tx/scheme_cache/scheme_cache.cpp @@ -2,129 +2,129 @@ #include <ydb/core/base/path.h> -#include <util/string/builder.h> +#include <util/string/builder.h> namespace NKikimr { namespace NSchemeCache { -TSchemeCacheConfig::TSchemeCacheConfig(const TAppData* appData, NMonitoring::TDynamicCounterPtr counters) - : Counters(counters) -{ - Y_VERIFY(appData); - Y_VERIFY(appData->DomainsInfo); +TSchemeCacheConfig::TSchemeCacheConfig(const TAppData* appData, NMonitoring::TDynamicCounterPtr counters) + : Counters(counters) +{ + Y_VERIFY(appData); + Y_VERIFY(appData->DomainsInfo); - for (const auto& [_, domain] : appData->DomainsInfo->Domains) { - Y_VERIFY(domain); + for (const auto& [_, domain] : appData->DomainsInfo->Domains) { + Y_VERIFY(domain); - if (!domain->SchemeRoot) { - continue; + if (!domain->SchemeRoot) { + continue; } - Roots.emplace_back(domain->DomainRootTag(), domain->SchemeRoot, domain->Name); + Roots.emplace_back(domain->DomainRootTag(), domain->SchemeRoot, domain->Name); } -} - -TString TDomainInfo::ToString() const { - return TStringBuilder() << "{" - << " DomainKey: " << DomainKey - << " ResourcesDomainKey: " << ResourcesDomainKey - << " Params { " << Params.ShortDebugString() << " }" - << " }"; } -TString TSchemeCacheNavigate::TEntry::ToString() const { - return TStringBuilder() << "{" - << " Path: " << JoinPath(Path) - << " TableId: " << TableId - << " RequestType: " << RequestType - << " Operation: " << Operation - << " RedirectRequired: " << (RedirectRequired ? "true" : "false") - << " ShowPrivatePath: " << (ShowPrivatePath ? "true" : "false") - << " SyncVersion: " << (SyncVersion ? "true" : "false") - << " Status: " << Status - << " Kind: " << Kind - << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>") - << " }"; +TString TDomainInfo::ToString() const { + return TStringBuilder() << "{" + << " DomainKey: " << DomainKey + << " ResourcesDomainKey: " << ResourcesDomainKey + << " Params { " << Params.ShortDebugString() << " }" + << " }"; } -TString TSchemeCacheNavigate::TEntry::ToString(const NScheme::TTypeRegistry& typeRegistry) const { - Y_UNUSED(typeRegistry); - return ToString(); +TString TSchemeCacheNavigate::TEntry::ToString() const { + return TStringBuilder() << "{" + << " Path: " << JoinPath(Path) + << " TableId: " << TableId + << " RequestType: " << RequestType + << " Operation: " << Operation + << " RedirectRequired: " << (RedirectRequired ? "true" : "false") + << " ShowPrivatePath: " << (ShowPrivatePath ? "true" : "false") + << " SyncVersion: " << (SyncVersion ? "true" : "false") + << " Status: " << Status + << " Kind: " << Kind + << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>") + << " }"; } -template <typename TResultSet> -static TString ResultSetToString(const TResultSet& rs, const NScheme::TTypeRegistry& typeRegistry) { - TStringBuilder out; +TString TSchemeCacheNavigate::TEntry::ToString(const NScheme::TTypeRegistry& typeRegistry) const { + Y_UNUSED(typeRegistry); + return ToString(); +} + +template <typename TResultSet> +static TString ResultSetToString(const TResultSet& rs, const NScheme::TTypeRegistry& typeRegistry) { + TStringBuilder out; - for (ui32 i = 0; i < rs.size(); ++i) { - if (i) { - out << ","; + for (ui32 i = 0; i < rs.size(); ++i) { + if (i) { + out << ","; } - out << rs.at(i).ToString(typeRegistry); + out << rs.at(i).ToString(typeRegistry); } - return out; + return out; } -TString TSchemeCacheNavigate::ToString(const NScheme::TTypeRegistry& typeRegistry) const { - return TStringBuilder() << "{" - << " ErrorCount: " << ErrorCount - << " DatabaseName: " << DatabaseName - << " DomainOwnerId: " << DomainOwnerId - << " ResultSet [" << ResultSetToString(ResultSet, typeRegistry) << "]" - << " }"; +TString TSchemeCacheNavigate::ToString(const NScheme::TTypeRegistry& typeRegistry) const { + return TStringBuilder() << "{" + << " ErrorCount: " << ErrorCount + << " DatabaseName: " << DatabaseName + << " DomainOwnerId: " << DomainOwnerId + << " ResultSet [" << ResultSetToString(ResultSet, typeRegistry) << "]" + << " }"; } -TString TSchemeCacheRequest::TEntry::ToString() const { - return TStringBuilder() << "{" - << " TableId: " << (KeyDescription ? ::ToString(KeyDescription->TableId.PathId) : "<moved>") - << " Access: " << Access - << " SyncVersion: " << (SyncVersion ? "true" : "false") - << " Status: " << Status - << " Kind: " << Kind - << " PartitionsCount: " << (KeyDescription ? ::ToString(KeyDescription->Partitions.size()) : "<moved>") - << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>") - << " }"; +TString TSchemeCacheRequest::TEntry::ToString() const { + return TStringBuilder() << "{" + << " TableId: " << (KeyDescription ? ::ToString(KeyDescription->TableId.PathId) : "<moved>") + << " Access: " << Access + << " SyncVersion: " << (SyncVersion ? "true" : "false") + << " Status: " << Status + << " Kind: " << Kind + << " PartitionsCount: " << (KeyDescription ? ::ToString(KeyDescription->Partitions.size()) : "<moved>") + << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>") + << " }"; } -TString TSchemeCacheRequest::TEntry::ToString(const NScheme::TTypeRegistry& typeRegistry) const { - TStringBuilder out; - out << "{" - << " TableId: " << (KeyDescription ? ::ToString(KeyDescription->TableId.PathId) : "<moved>") - << " Access: " << Access - << " SyncVersion: " << (SyncVersion ? "true" : "false") - << " Status: " << Status - << " Kind: " << Kind - << " PartitionsCount: " << (KeyDescription ? ::ToString(KeyDescription->Partitions.size()) : "<moved>") - << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>"); - - if (KeyDescription) { - TDbTupleRef from(KeyDescription->KeyColumnTypes.data(), KeyDescription->Range.From.data(), KeyDescription->Range.From.size()); - TDbTupleRef to(KeyDescription->KeyColumnTypes.data(), KeyDescription->Range.To.data(), KeyDescription->Range.To.size()); - - if (KeyDescription->Range.Point) { - out << " Point: " << DbgPrintTuple(from, typeRegistry); +TString TSchemeCacheRequest::TEntry::ToString(const NScheme::TTypeRegistry& typeRegistry) const { + TStringBuilder out; + out << "{" + << " TableId: " << (KeyDescription ? ::ToString(KeyDescription->TableId.PathId) : "<moved>") + << " Access: " << Access + << " SyncVersion: " << (SyncVersion ? "true" : "false") + << " Status: " << Status + << " Kind: " << Kind + << " PartitionsCount: " << (KeyDescription ? ::ToString(KeyDescription->Partitions.size()) : "<moved>") + << " DomainInfo " << (DomainInfo ? DomainInfo->ToString() : "<null>"); + + if (KeyDescription) { + TDbTupleRef from(KeyDescription->KeyColumnTypes.data(), KeyDescription->Range.From.data(), KeyDescription->Range.From.size()); + TDbTupleRef to(KeyDescription->KeyColumnTypes.data(), KeyDescription->Range.To.data(), KeyDescription->Range.To.size()); + + if (KeyDescription->Range.Point) { + out << " Point: " << DbgPrintTuple(from, typeRegistry); } else { - out << " From: " << DbgPrintTuple(from, typeRegistry) - << " IncFrom: " << KeyDescription->Range.InclusiveFrom - << " To: " << DbgPrintTuple(to, typeRegistry) - << " IncTo: " << KeyDescription->Range.InclusiveTo; + out << " From: " << DbgPrintTuple(from, typeRegistry) + << " IncFrom: " << KeyDescription->Range.InclusiveFrom + << " To: " << DbgPrintTuple(to, typeRegistry) + << " IncTo: " << KeyDescription->Range.InclusiveTo; } } - out << " }"; - return out; + out << " }"; + return out; } -TString TSchemeCacheRequest::ToString(const NScheme::TTypeRegistry& typeRegistry) const { - return TStringBuilder() << "{" - << " ErrorCount: " << ErrorCount - << " DatabaseName: " << DatabaseName - << " DomainOwnerId: " << DomainOwnerId - << " ResultSet [" << ResultSetToString(ResultSet, typeRegistry) << "]" - << " }"; +TString TSchemeCacheRequest::ToString(const NScheme::TTypeRegistry& typeRegistry) const { + return TStringBuilder() << "{" + << " ErrorCount: " << ErrorCount + << " DatabaseName: " << DatabaseName + << " DomainOwnerId: " << DomainOwnerId + << " ResultSet [" << ResultSetToString(ResultSet, typeRegistry) << "]" + << " }"; } -} // NSchemeCache -} // NKikimr +} // NSchemeCache +} // NKikimr diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 576cd1d7208..26a07897a11 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -12,7 +12,7 @@ #include <ydb/core/tx/datashard/sys_tables.h> #include <ydb/library/aclib/aclib.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/hash.h> #include <util/generic/hash_set.h> #include <util/generic/ptr.h> @@ -22,20 +22,20 @@ namespace NSchemeCache { struct TSchemeCacheConfig : public TThrRefBase { struct TTagEntry { - ui32 Tag = 0; - ui64 RootSchemeShard = 0; + ui32 Tag = 0; + ui64 RootSchemeShard = 0; TString Name; - explicit TTagEntry(ui32 tag, ui64 rootSchemeShard, const TString& name) + explicit TTagEntry(ui32 tag, ui64 rootSchemeShard, const TString& name) : Tag(tag) , RootSchemeShard(rootSchemeShard) , Name(name) {} }; - TSchemeCacheConfig() = default; - explicit TSchemeCacheConfig(const TAppData* appData, NMonitoring::TDynamicCounterPtr counters); - + TSchemeCacheConfig() = default; + explicit TSchemeCacheConfig(const TAppData* appData, NMonitoring::TDynamicCounterPtr counters); + TVector<TTagEntry> Roots; NMonitoring::TDynamicCounterPtr Counters; }; @@ -43,61 +43,61 @@ struct TSchemeCacheConfig : public TThrRefBase { struct TDomainInfo : public TAtomicRefCount<TDomainInfo> { using TPtr = TIntrusivePtr<TDomainInfo>; - explicit TDomainInfo(const TPathId& domainKey, const TPathId& resourcesDomainKey) + explicit TDomainInfo(const TPathId& domainKey, const TPathId& resourcesDomainKey) : DomainKey(domainKey) - , ResourcesDomainKey(resourcesDomainKey) + , ResourcesDomainKey(resourcesDomainKey) , Coordinators(TVector<ui64>{}) {} - explicit TDomainInfo(const NKikimrSubDomains::TDomainDescription& descr) - : DomainKey(GetDomainKey(descr.GetDomainKey())) + explicit TDomainInfo(const NKikimrSubDomains::TDomainDescription& descr) + : DomainKey(GetDomainKey(descr.GetDomainKey())) , Params(descr.GetProcessingParams()) , Coordinators(descr.GetProcessingParams()) - { - if (descr.HasResourcesDomainKey()) { - ResourcesDomainKey = GetDomainKey(descr.GetResourcesDomainKey()); - } else { - ResourcesDomainKey = DomainKey; - } - } - - inline ui64 GetVersion() const { + { + if (descr.HasResourcesDomainKey()) { + ResourcesDomainKey = GetDomainKey(descr.GetResourcesDomainKey()); + } else { + ResourcesDomainKey = DomainKey; + } + } + + inline ui64 GetVersion() const { return Params.GetVersion(); } - inline ui64 ExtractSchemeShard() const { - if (Params.HasSchemeShard()) { - return Params.GetSchemeShard(); - } else { - return DomainKey.OwnerId; - } - } - - TPathId DomainKey; - TPathId ResourcesDomainKey; + inline ui64 ExtractSchemeShard() const { + if (Params.HasSchemeShard()) { + return Params.GetSchemeShard(); + } else { + return DomainKey.OwnerId; + } + } + + TPathId DomainKey; + TPathId ResourcesDomainKey; NKikimrSubDomains::TProcessingParams Params; TCoordinators Coordinators; - - TString ToString() const; - -private: - inline static TPathId GetDomainKey(const NKikimrSubDomains::TDomainKey& protoKey) { - return TPathId(protoKey.GetSchemeShard(), protoKey.GetPathId()); - } - -}; // TDomainInfo - + + TString ToString() const; + +private: + inline static TPathId GetDomainKey(const NKikimrSubDomains::TDomainKey& protoKey) { + return TPathId(protoKey.GetSchemeShard(), protoKey.GetPathId()); + } + +}; // TDomainInfo + struct TSchemeCacheNavigate { enum class EStatus { - Unknown = 0, - RootUnknown = 1, - PathErrorUnknown = 2, - PathNotTable = 3, - PathNotPath = 4, - TableCreationNotComplete = 5, - LookupError = 6, - RedirectLookupError = 7, - Ok = 128, + Unknown = 0, + RootUnknown = 1, + PathErrorUnknown = 2, + PathNotTable = 3, + PathNotPath = 4, + TableCreationNotComplete = 5, + LookupError = 6, + RedirectLookupError = 7, + Ok = 128, }; enum EOp { @@ -121,27 +121,27 @@ struct TSchemeCacheNavigate { KindIndex = 10, KindOlapStore = 11, KindOlapTable = 12, - KindCdcStream = 13, + KindCdcStream = 13, KindSequence = 14, - KindReplication = 15, + KindReplication = 15, }; struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> { struct TChild { const TString Name; - const TPathId PathId; + const TPathId PathId; const ui64 SchemaVersion; const EKind Kind; - - explicit TChild(const TString& name, const TPathId& pathId, EKind kind, ui64 schemaVersion = 0) - : Name(name) + + explicit TChild(const TString& name, const TPathId& pathId, EKind kind, ui64 schemaVersion = 0) + : Name(name) , PathId(pathId) - , SchemaVersion(schemaVersion) - , Kind(kind) - {} + , SchemaVersion(schemaVersion) + , Kind(kind) + {} }; - - EKind Kind = KindUnknown; + + EKind Kind = KindUnknown; TVector<TChild> Children; }; @@ -185,80 +185,80 @@ struct TSchemeCacheNavigate { TTableId OlapStoreId; }; - struct TCdcStreamInfo : public TAtomicRefCount<TCdcStreamInfo> { - EKind Kind = KindUnknown; + struct TCdcStreamInfo : public TAtomicRefCount<TCdcStreamInfo> { + EKind Kind = KindUnknown; NKikimrSchemeOp::TCdcStreamDescription Description; - TPathId PathId; - }; - + TPathId PathId; + }; + struct TSequenceInfo : public TAtomicRefCount<TSequenceInfo> { EKind Kind = KindUnknown; NKikimrSchemeOp::TSequenceDescription Description; }; - struct TReplicationInfo : public TAtomicRefCount<TReplicationInfo> { - EKind Kind = KindUnknown; - NKikimrSchemeOp::TReplicationDescription Description; - }; - + struct TReplicationInfo : public TAtomicRefCount<TReplicationInfo> { + EKind Kind = KindUnknown; + NKikimrSchemeOp::TReplicationDescription Description; + }; + struct TEntry { enum class ERequestType : ui8 { ByPath, ByTableId }; - - // in - TVector<TString> Path; - TTableId TableId; + + // in + TVector<TString> Path; + TTableId TableId; ERequestType RequestType = ERequestType::ByPath; - EOp Operation = OpUnknown; + EOp Operation = OpUnknown; bool RedirectRequired = true; - bool ShowPrivatePath = false; - bool SyncVersion = false; + bool ShowPrivatePath = false; + bool SyncVersion = false; - // out + // out EStatus Status = EStatus::Unknown; - EKind Kind = KindUnknown; - ui64 CreateStep = 0; + EKind Kind = KindUnknown; + ui64 CreateStep = 0; - // common - TIntrusivePtr<TDomainInfo> DomainInfo; - TIntrusivePtr<TSecurityObject> SecurityObject; - TIntrusiveConstPtr<TDirEntryInfo> Self; - TIntrusiveConstPtr<TListNodeEntry> ListNodeEntry; + // common + TIntrusivePtr<TDomainInfo> DomainInfo; + TIntrusivePtr<TSecurityObject> SecurityObject; + TIntrusiveConstPtr<TDirEntryInfo> Self; + TIntrusiveConstPtr<TListNodeEntry> ListNodeEntry; - // table specific - THashMap<TString, TString> Attributes; + // table specific + THashMap<TString, TString> Attributes; THashMap<ui32, TSysTables::TTableColumnInfo> Columns; THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; - // other + // other TIntrusiveConstPtr<TDomainDescription> DomainDescription; TIntrusiveConstPtr<TPQGroupInfo> PQGroupInfo; TIntrusiveConstPtr<TRtmrVolumeInfo> RTMRVolumeInfo; TIntrusiveConstPtr<TKesusInfo> KesusInfo; - TIntrusiveConstPtr<TSolomonVolumeInfo> SolomonVolumeInfo; + TIntrusiveConstPtr<TSolomonVolumeInfo> SolomonVolumeInfo; TIntrusiveConstPtr<TOlapStoreInfo> OlapStoreInfo; TIntrusiveConstPtr<TOlapTableInfo> OlapTableInfo; - TIntrusiveConstPtr<TCdcStreamInfo> CdcStreamInfo; + TIntrusiveConstPtr<TCdcStreamInfo> CdcStreamInfo; TIntrusiveConstPtr<TSequenceInfo> SequenceInfo; - TIntrusiveConstPtr<TReplicationInfo> ReplicationInfo; + TIntrusiveConstPtr<TReplicationInfo> ReplicationInfo; - TString ToString() const; - TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; + TString ToString() const; + TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; }; using TResultSet = TVector<TEntry>; TResultSet ResultSet; - TAutoPtr<const NACLib::TUserToken> UserToken; - TString DatabaseName; - ui64 DomainOwnerId = 0; - ui64 ErrorCount = 0; - const ui64 Instant; // deprecated, used by pq - + TAutoPtr<const NACLib::TUserToken> UserToken; + TString DatabaseName; + ui64 DomainOwnerId = 0; + ui64 ErrorCount = 0; + const ui64 Instant; // deprecated, used by pq + TSchemeCacheNavigate() : Instant(0) {} @@ -267,20 +267,20 @@ struct TSchemeCacheNavigate { : Instant(instant) {} - TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; - -}; // TSchemeCacheNavigate - + TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; + +}; // TSchemeCacheNavigate + struct TSchemeCacheRequest { enum class EStatus { - Unknown = 0, - TypeCheckError = 1, - NotMaterialized = 2, - PathErrorUnknown = 3, - PathErrorNotExist = 4, - LookupError = 5, - OkScheme = 128, - OkData = 129, + Unknown = 0, + TypeCheckError = 1, + NotMaterialized = 2, + PathErrorUnknown = 3, + PathErrorNotExist = 4, + LookupError = 5, + OkScheme = 128, + OkData = 129, }; enum EOp { @@ -290,74 +290,74 @@ struct TSchemeCacheRequest { OpScheme = 1 << 3, }; - enum EKind { - KindUnknown = 0, - KindRegularTable = 1, - KindSyncIndexTable = 2, - KindAsyncIndexTable= 3, - }; - + enum EKind { + KindUnknown = 0, + KindRegularTable = 1, + KindSyncIndexTable = 2, + KindAsyncIndexTable= 3, + }; + struct TEntry { - // in + // in THolder<TKeyDesc> KeyDescription; - ui32 Access = 0; + ui32 Access = 0; uintptr_t UserData = 0; - bool SyncVersion = false; - - // out - EStatus Status = EStatus::Unknown; - EKind Kind = EKind::KindUnknown; - TIntrusivePtr<TDomainInfo> DomainInfo; - - explicit TEntry(THolder<TKeyDesc> keyDesc) - : KeyDescription(std::move(keyDesc)) + bool SyncVersion = false; + + // out + EStatus Status = EStatus::Unknown; + EKind Kind = EKind::KindUnknown; + TIntrusivePtr<TDomainInfo> DomainInfo; + + explicit TEntry(THolder<TKeyDesc> keyDesc) + : KeyDescription(std::move(keyDesc)) { Y_VERIFY_DEBUG(KeyDescription); } - TEntry(TKeyDesc* keyDesc) = delete; + TEntry(TKeyDesc* keyDesc) = delete; - TString ToString() const; - TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; + TString ToString() const; + TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; }; using TResultSet = TVector<TEntry>; TResultSet ResultSet; - TAutoPtr<const NACLib::TUserToken> UserToken; - TString DatabaseName; - ui64 DomainOwnerId = 0; - ui64 ErrorCount = 0; - - TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; - -}; // TSchemeCacheRequest - -struct TSchemeCacheRequestContext : TAtomicRefCount<TSchemeCacheRequestContext>, TNonCopyable { + TAutoPtr<const NACLib::TUserToken> UserToken; + TString DatabaseName; + ui64 DomainOwnerId = 0; + ui64 ErrorCount = 0; + + TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; + +}; // TSchemeCacheRequest + +struct TSchemeCacheRequestContext : TAtomicRefCount<TSchemeCacheRequestContext>, TNonCopyable { TActorId Sender; ui64 WaitCounter; - TAutoPtr<TSchemeCacheRequest> Request; - const TInstant CreatedAt; + TAutoPtr<TSchemeCacheRequest> Request; + const TInstant CreatedAt; - TSchemeCacheRequestContext(const TActorId& sender, TAutoPtr<TSchemeCacheRequest> request, const TInstant& now = TInstant::Now()) + TSchemeCacheRequestContext(const TActorId& sender, TAutoPtr<TSchemeCacheRequest> request, const TInstant& now = TInstant::Now()) : Sender(sender) , WaitCounter(0) , Request(request) - , CreatedAt(now) + , CreatedAt(now) {} }; -struct TSchemeCacheNavigateContext : TAtomicRefCount<TSchemeCacheNavigateContext>, TNonCopyable { +struct TSchemeCacheNavigateContext : TAtomicRefCount<TSchemeCacheNavigateContext>, TNonCopyable { TActorId Sender; ui64 WaitCounter; - TAutoPtr<TSchemeCacheNavigate> Request; - const TInstant CreatedAt; + TAutoPtr<TSchemeCacheNavigate> Request; + const TInstant CreatedAt; - TSchemeCacheNavigateContext(const TActorId& sender, TAutoPtr<TSchemeCacheNavigate> request, const TInstant& now = TInstant::Now()) + TSchemeCacheNavigateContext(const TActorId& sender, TAutoPtr<TSchemeCacheNavigate> request, const TInstant& now = TInstant::Now()) : Sender(sender) , WaitCounter(0) , Request(request) - , CreatedAt(now) + , CreatedAt(now) {} }; @@ -397,12 +397,12 @@ public: } }; -} // NSchemeCache +} // NSchemeCache struct TEvTxProxySchemeCache { enum EEv { - EvResolveKeySet = EventSpaceBegin(TKikimrEvents::ES_SCHEME_CACHE), - EvInvalidateDistEntry, // unused + EvResolveKeySet = EventSpaceBegin(TKikimrEvents::ES_SCHEME_CACHE), + EvInvalidateDistEntry, // unused EvResolveKeySetResult, EvNavigateKeySet, EvNavigateKeySetResult, @@ -417,33 +417,33 @@ struct TEvTxProxySchemeCache { EvEnd, }; - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_CACHE), "expect EvEnd < EventSpaceEnd(ES_SCHEME_CACHE)"); + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_SCHEME_CACHE), "expect EvEnd < EventSpaceEnd(ES_SCHEME_CACHE)"); -private: - template <typename TDerived, ui32 EventType, typename TRequest> - struct TEvBasic : public TEventLocal<TDerived, EventType> { - TAutoPtr<TRequest> Request; +private: + template <typename TDerived, ui32 EventType, typename TRequest> + struct TEvBasic : public TEventLocal<TDerived, EventType> { + TAutoPtr<TRequest> Request; - TEvBasic(TAutoPtr<TRequest> request) + TEvBasic(TAutoPtr<TRequest> request) : Request(request) {} }; -public: - struct TEvResolveKeySet : public TEvBasic<TEvResolveKeySet, EvResolveKeySet, NSchemeCache::TSchemeCacheRequest> { - using TEvBasic::TEvBasic; - }; +public: + struct TEvResolveKeySet : public TEvBasic<TEvResolveKeySet, EvResolveKeySet, NSchemeCache::TSchemeCacheRequest> { + using TEvBasic::TEvBasic; + }; - struct TEvResolveKeySetResult : public TEvBasic<TEvResolveKeySetResult, EvResolveKeySetResult, NSchemeCache::TSchemeCacheRequest> { - using TEvBasic::TEvBasic; + struct TEvResolveKeySetResult : public TEvBasic<TEvResolveKeySetResult, EvResolveKeySetResult, NSchemeCache::TSchemeCacheRequest> { + using TEvBasic::TEvBasic; }; - struct TEvNavigateKeySet : public TEvBasic<TEvNavigateKeySet, EvNavigateKeySet, NSchemeCache::TSchemeCacheNavigate> { - using TEvBasic::TEvBasic; + struct TEvNavigateKeySet : public TEvBasic<TEvNavigateKeySet, EvNavigateKeySet, NSchemeCache::TSchemeCacheNavigate> { + using TEvBasic::TEvBasic; }; struct TEvNavigateKeySetResult : public TEvBasic<TEvNavigateKeySetResult, EvNavigateKeySetResult, NSchemeCache::TSchemeCacheNavigate> { - using TEvBasic::TEvBasic; + using TEvBasic::TEvBasic; }; struct TEvInvalidateTable : public TEventLocal<TEvInvalidateTable, EvInvalidateTable> { @@ -483,14 +483,14 @@ public: }; struct TEvWatchNotifyUpdated : public TEventLocal<TEvWatchNotifyUpdated, EvWatchNotifyUpdated> { - using TDescribeResult = NSchemeCache::TDescribeResult; - + using TDescribeResult = NSchemeCache::TDescribeResult; + const ui64 Key; const TString Path; const TPathId PathId; - TDescribeResult::TCPtr Result; + TDescribeResult::TCPtr Result; - TEvWatchNotifyUpdated(ui64 key, const TString& path, const TPathId& pathId, TDescribeResult::TCPtr result) + TEvWatchNotifyUpdated(ui64 key, const TString& path, const TPathId& pathId, TDescribeResult::TCPtr result) : Key(key) , Path(path) , PathId(pathId) @@ -527,4 +527,4 @@ inline TActorId MakeSchemeCacheID() { return TActorId(0, TStringBuf("SchmCcheSrv")); } -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/scheme_cache/ya.make b/ydb/core/tx/scheme_cache/ya.make index 1d5f45d4c93..c9ead108c08 100644 --- a/ydb/core/tx/scheme_cache/ya.make +++ b/ydb/core/tx/scheme_cache/ya.make @@ -17,8 +17,8 @@ SRCS( scheme_cache.cpp ) -GENERATE_ENUM_SERIALIZATION(scheme_cache.h) - -YQL_LAST_ABI_VERSION() - +GENERATE_ENUM_SERIALIZATION(scheme_cache.h) + +YQL_LAST_ABI_VERSION() + END() diff --git a/ydb/core/tx/schemeshard/schemeshard.cpp b/ydb/core/tx/schemeshard/schemeshard.cpp index d61c953caf9..24b73f590cc 100644 --- a/ydb/core/tx/schemeshard/schemeshard.cpp +++ b/ydb/core/tx/schemeshard/schemeshard.cpp @@ -10,7 +10,7 @@ namespace NSchemeShard { TString owner, TString effectiveRootACL, ui64 effectiveRootACLVersion, const NKikimrSubDomains::TProcessingParams &processingParams, const TStoragePools &storagePools, const TMap<TString, TString> userAttrData, ui64 UserAttrsVersion, const NSchemeShard::TSchemeLimits &limits, - ui64 sharedHive, const TPathId& resourcesDomainId) + ui64 sharedHive, const TPathId& resourcesDomainId) { Record.SetDomainSchemeShard(selfTabletId); Record.SetDomainPathId(pathId); @@ -33,15 +33,15 @@ namespace NSchemeShard { Record.SetUserAttributesVersion(UserAttrsVersion); *Record.MutableSchemeLimits() = limits.AsProto(); - - if (sharedHive != ui64(InvalidTabletId)) { - Record.SetSharedHive(sharedHive); - } - - if (resourcesDomainId) { - Record.SetResourcesDomainOwnerId(resourcesDomainId.OwnerId); - Record.SetResourcesDomainPathId(resourcesDomainId.LocalPathId); - } + + if (sharedHive != ui64(InvalidTabletId)) { + Record.SetSharedHive(sharedHive); + } + + if (resourcesDomainId) { + Record.SetResourcesDomainOwnerId(resourcesDomainId.OwnerId); + Record.SetResourcesDomainPathId(resourcesDomainId.LocalPathId); + } } } diff --git a/ydb/core/tx/schemeshard/schemeshard.h b/ydb/core/tx/schemeshard/schemeshard.h index e80254c4c00..d33173508ce 100644 --- a/ydb/core/tx/schemeshard/schemeshard.h +++ b/ydb/core/tx/schemeshard/schemeshard.h @@ -58,8 +58,8 @@ struct TEvSchemeShard { EvBackupDatashard = EvModifySchemeTransaction + 6 * 512, EvBackupDatashardResult, - EvCancelTx, - EvCancelTxResult, + EvCancelTx, + EvCancelTxResult, EvEnd }; @@ -92,21 +92,21 @@ struct TEvSchemeShard { } }; - struct TEvCancelTx - : public TEventPB<TEvCancelTx, + struct TEvCancelTx + : public TEventPB<TEvCancelTx, NKikimrScheme::TEvCancelTx, - EvCancelTx> + EvCancelTx> { }; - struct TEvCancelTxResult: - public TEventPB<TEvCancelTxResult, + struct TEvCancelTxResult: + public TEventPB<TEvCancelTxResult, NKikimrScheme::TEvCancelTxResult, - EvCancelTxResult> + EvCancelTxResult> { - TEvCancelTxResult() = default; - TEvCancelTxResult(ui64 targetTxId, ui64 txId) { - Record.SetTargetTxId(targetTxId); + TEvCancelTxResult() = default; + TEvCancelTxResult(ui64 targetTxId, ui64 txId) { + Record.SetTargetTxId(targetTxId); Record.SetTxId(txId); } }; @@ -316,7 +316,7 @@ struct TEvSchemeShard { const NKikimrSubDomains::TProcessingParams& processingParams, const TStoragePools& storagePools, const TMap<TString, TString> userAttrData, ui64 UserAttrsVersion, - const TSchemeLimits& limits, ui64 sharedHive, const TPathId& resourcesDomainId = TPathId()); + const TSchemeLimits& limits, ui64 sharedHive, const TPathId& resourcesDomainId = TPathId()); }; struct TEvInitTenantSchemeShardResult: public TEventPB<TEvInitTenantSchemeShardResult, diff --git a/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp b/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp index 37e9787a98f..d804955b69b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp @@ -1,471 +1,471 @@ -#include "schemeshard_impl.h" - -#include <util/string/join.h> - -namespace NKikimr { +#include "schemeshard_impl.h" + +#include <util/string/join.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TTxRunConditionalErase: public TSchemeShard::TRwTxBase { - TTableInfo::TPtr TableInfo; - THashMap<TTabletId, NKikimrTxDataShard::TEvConditionalEraseRowsRequest> RunOnTablets; - - TTxRunConditionalErase(TSelf *self, TEvPrivate::TEvRunConditionalErase::TPtr& ev) - : TRwTxBase(self) - , TableInfo(nullptr) - { - Y_UNUSED(ev); - } - - TTxRunConditionalErase(TSelf *self, TTableInfo::TPtr tableInfo) - : TRwTxBase(self) - , TableInfo(tableInfo) - { - } - - TTxType GetTxType() const override { - return TXTYPE_RUN_CONDITIONAL_ERASE; - } - - void DoExecute(TTransactionContext&, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxRunConditionalErase DoExecute" - << ": at schemeshard: " << Self->TabletID()); - - if (!Self->AllowConditionalEraseOperations) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase operations are not allowed" - << ", skip TTxRunConditionalErase" - << ": at schemeshard: " << Self->TabletID()); - return; - } - - if (!TableInfo) { - for (const auto& [_, tableInfo] : Self->TTLEnabledTables) { - DoExecuteOnTable(tableInfo, ctx); - } - } else { - DoExecuteOnTable(TableInfo, ctx); - } - } - - void DoExecuteOnTable(TTableInfo::TPtr tableInfo, const TActorContext& ctx) { - if (!tableInfo->IsTTLEnabled()) { - LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P1" - << ", at schemeshard: " << Self->TabletID()); - return; - } - - const auto& partitions = tableInfo->GetPartitions(); - const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); - THashSet<TShardIdx> reschedule; - - for (const auto& shardIdx : tableInfo->GetInFlightCondErase()) { - Y_VERIFY(shardToPartition.contains(shardIdx)); - const ui64 partitionIdx = shardToPartition.at(shardIdx); - - Y_VERIFY(partitionIdx < partitions.size()); - const auto& tableShardInfo = partitions.at(partitionIdx); - - if (!DoExecuteOnShard(tableInfo, tableShardInfo, ctx)) { - reschedule.insert(shardIdx); - } - } - - for (const auto& shardIdx : reschedule) { - tableInfo->RescheduleCondErase(shardIdx); - } - - const auto& sysSettings = tableInfo->TTLSettings().GetEnabled().GetSysSettings(); - const auto maxInFlight = sysSettings.GetMaxShardsInFlight(); - - while (true) { - if (maxInFlight && tableInfo->GetInFlightCondErase().size() >= maxInFlight) { - break; - } - - const auto* tableShardInfo = tableInfo->GetScheduledCondEraseShard(); - if (!tableShardInfo) { - break; - } - - if (!DoExecuteOnShard(tableInfo, *tableShardInfo, ctx)) { - break; - } - - tableInfo->AddInFlightCondErase(tableShardInfo->ShardIdx); - } - } - - bool DoExecuteOnShard(TTableInfo::TPtr tableInfo, const TTableShardInfo& tableShardInfo, const TActorContext& ctx) { - if (tableShardInfo.NextCondErase > ctx.Now()) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Skip conditional erase" - << ": shardIdx: " << tableShardInfo.ShardIdx - << ", run at: " << tableShardInfo.NextCondErase - << ", at schemeshard: " << Self->TabletID()); - return false; - } - - if (!Self->ShardInfos.contains(tableShardInfo.ShardIdx)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" - << ": shardIdx: " << tableShardInfo.ShardIdx - << ", at schemeshard: " << Self->TabletID()); - return false; - } - - const TShardInfo& shardInfo = Self->ShardInfos.at(tableShardInfo.ShardIdx); - if (!Self->PathsById.contains(shardInfo.PathId)) { - LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve path" - << ": shardIdx: " << tableShardInfo.ShardIdx - << ": pathId: " << shardInfo.PathId - << ", at schemeshard: " << Self->TabletID()); - return false; - } - - auto path = Self->PathsById.at(shardInfo.PathId); - if (path->Dropped()) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Table is dropped" - << ": shardIdx: " << tableShardInfo.ShardIdx - << ": pathId: " << shardInfo.PathId - << ", at schemeshard: " << Self->TabletID()); - return false; - } - - if (!Self->Tables.contains(shardInfo.PathId)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve table" - << ": shardIdx: " << tableShardInfo.ShardIdx - << ": pathId: " << shardInfo.PathId - << ", at schemeshard: " << Self->TabletID()); - return false; - } - - const auto& settings = tableInfo->TTLSettings().GetEnabled(); - const TDuration expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds()); - const TInstant wallClock = ctx.Now() - expireAfter; - - NKikimrTxDataShard::TEvConditionalEraseRowsRequest request; - request.SetTableId(shardInfo.PathId.LocalPathId); - request.SetSchemaVersion(tableInfo->AlterVersion); - - for (const auto& [indexId, columnIds]: MakeIndexes(shardInfo.PathId)) { - auto& index = *request.MutableIndexes()->Add(); - - index.SetOwnerId(indexId.PathId.OwnerId); - index.SetPathId(indexId.PathId.LocalPathId); - index.SetSchemaVersion(indexId.SchemaVersion); - - for (const auto& [indexColumnId, mainColumnId] : columnIds) { - auto& keyMap = *index.MutableKeyMap()->Add(); - keyMap.SetIndexColumnId(indexColumnId); - keyMap.SetMainColumnId(mainColumnId); - } - } - - request.MutableExpiration()->SetColumnId(tableInfo->GetTTLColumnId()); - request.MutableExpiration()->SetWallClockTimestamp(wallClock.GetValue()); - request.MutableExpiration()->SetColumnUnit(settings.GetColumnUnit()); - - const auto& sysSettings = settings.GetSysSettings(); - request.MutableLimits()->SetBatchMaxBytes(sysSettings.GetBatchMaxBytes()); - request.MutableLimits()->SetBatchMinKeys(sysSettings.GetBatchMinKeys()); - request.MutableLimits()->SetBatchMaxKeys(sysSettings.GetBatchMaxKeys()); - - RunOnTablets.emplace(shardInfo.TabletID, std::move(request)); - - return true; - } - - void DoComplete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxRunConditionalErase DoComplete" - << ": at schemeshard: " << Self->TabletID()); - - for (auto& kv : RunOnTablets) { - const auto& tabletId = kv.first; - auto& request = kv.second; - - auto ev = MakeHolder<TEvDataShard::TEvConditionalEraseRowsRequest>(); - ev->Record = std::move(request); - - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Run conditional erase" - << ", tabletId: " << tabletId - << ", request: " << ev->Record.ShortDebugString() - << ", at schemeshard: " << Self->TabletID()); - Self->PipeClientCache->Send(ctx, ui64(tabletId), ev.Release()); - } - - if (!TableInfo) { - Self->ScheduleConditionalEraseRun(ctx); - } - } - -private: - THashMap<TTableId, TVector<std::pair<ui32, ui32>>> MakeIndexes(const TPathId& mainPathId) const { - THashMap<TTableId, TVector<std::pair<ui32, ui32>>> result; - - Y_VERIFY(Self->PathsById.contains(mainPathId)); - auto mainPath = Self->PathsById.at(mainPathId); - - Y_VERIFY(Self->Tables.contains(mainPathId)); - auto mainTable = Self->Tables.at(mainPathId); - - for (const auto& [_, childPathId] : mainPath->GetChildren()) { - Y_VERIFY(Self->PathsById.contains(childPathId)); - auto childPath = Self->PathsById.at(childPathId); - - if (!childPath->IsTableIndex() || childPath->Dropped()) { - continue; - } - - auto index = GetIndex(childPath); - if (index->Type == NKikimrSchemeOp::EIndexTypeGlobalAsync) { - continue; - } - - auto [indexImplTableId, indexImplTable] = GetIndexImplTable(childPath); - - auto ret = result.emplace( - TTableId(indexImplTableId, indexImplTable->AlterVersion), - MakeColumnIds(mainTable, index, indexImplTable) - ); - Y_VERIFY(ret.second); - } - - return result; - } - - TTableIndexInfo::TPtr GetIndex(TPathElement::TPtr indexPath) const { - Y_VERIFY(Self->Indexes.contains(indexPath->PathId)); - return Self->Indexes.at(indexPath->PathId); - } - - std::pair<TPathId, TTableInfo::TPtr> GetIndexImplTable(TPathElement::TPtr indexPath) const { - Y_VERIFY(indexPath->GetChildren().size() == 1); - - for (const auto& [_, indexImplPathId] : indexPath->GetChildren()) { + TTableInfo::TPtr TableInfo; + THashMap<TTabletId, NKikimrTxDataShard::TEvConditionalEraseRowsRequest> RunOnTablets; + + TTxRunConditionalErase(TSelf *self, TEvPrivate::TEvRunConditionalErase::TPtr& ev) + : TRwTxBase(self) + , TableInfo(nullptr) + { + Y_UNUSED(ev); + } + + TTxRunConditionalErase(TSelf *self, TTableInfo::TPtr tableInfo) + : TRwTxBase(self) + , TableInfo(tableInfo) + { + } + + TTxType GetTxType() const override { + return TXTYPE_RUN_CONDITIONAL_ERASE; + } + + void DoExecute(TTransactionContext&, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxRunConditionalErase DoExecute" + << ": at schemeshard: " << Self->TabletID()); + + if (!Self->AllowConditionalEraseOperations) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase operations are not allowed" + << ", skip TTxRunConditionalErase" + << ": at schemeshard: " << Self->TabletID()); + return; + } + + if (!TableInfo) { + for (const auto& [_, tableInfo] : Self->TTLEnabledTables) { + DoExecuteOnTable(tableInfo, ctx); + } + } else { + DoExecuteOnTable(TableInfo, ctx); + } + } + + void DoExecuteOnTable(TTableInfo::TPtr tableInfo, const TActorContext& ctx) { + if (!tableInfo->IsTTLEnabled()) { + LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P1" + << ", at schemeshard: " << Self->TabletID()); + return; + } + + const auto& partitions = tableInfo->GetPartitions(); + const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); + THashSet<TShardIdx> reschedule; + + for (const auto& shardIdx : tableInfo->GetInFlightCondErase()) { + Y_VERIFY(shardToPartition.contains(shardIdx)); + const ui64 partitionIdx = shardToPartition.at(shardIdx); + + Y_VERIFY(partitionIdx < partitions.size()); + const auto& tableShardInfo = partitions.at(partitionIdx); + + if (!DoExecuteOnShard(tableInfo, tableShardInfo, ctx)) { + reschedule.insert(shardIdx); + } + } + + for (const auto& shardIdx : reschedule) { + tableInfo->RescheduleCondErase(shardIdx); + } + + const auto& sysSettings = tableInfo->TTLSettings().GetEnabled().GetSysSettings(); + const auto maxInFlight = sysSettings.GetMaxShardsInFlight(); + + while (true) { + if (maxInFlight && tableInfo->GetInFlightCondErase().size() >= maxInFlight) { + break; + } + + const auto* tableShardInfo = tableInfo->GetScheduledCondEraseShard(); + if (!tableShardInfo) { + break; + } + + if (!DoExecuteOnShard(tableInfo, *tableShardInfo, ctx)) { + break; + } + + tableInfo->AddInFlightCondErase(tableShardInfo->ShardIdx); + } + } + + bool DoExecuteOnShard(TTableInfo::TPtr tableInfo, const TTableShardInfo& tableShardInfo, const TActorContext& ctx) { + if (tableShardInfo.NextCondErase > ctx.Now()) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Skip conditional erase" + << ": shardIdx: " << tableShardInfo.ShardIdx + << ", run at: " << tableShardInfo.NextCondErase + << ", at schemeshard: " << Self->TabletID()); + return false; + } + + if (!Self->ShardInfos.contains(tableShardInfo.ShardIdx)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" + << ": shardIdx: " << tableShardInfo.ShardIdx + << ", at schemeshard: " << Self->TabletID()); + return false; + } + + const TShardInfo& shardInfo = Self->ShardInfos.at(tableShardInfo.ShardIdx); + if (!Self->PathsById.contains(shardInfo.PathId)) { + LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve path" + << ": shardIdx: " << tableShardInfo.ShardIdx + << ": pathId: " << shardInfo.PathId + << ", at schemeshard: " << Self->TabletID()); + return false; + } + + auto path = Self->PathsById.at(shardInfo.PathId); + if (path->Dropped()) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Table is dropped" + << ": shardIdx: " << tableShardInfo.ShardIdx + << ": pathId: " << shardInfo.PathId + << ", at schemeshard: " << Self->TabletID()); + return false; + } + + if (!Self->Tables.contains(shardInfo.PathId)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve table" + << ": shardIdx: " << tableShardInfo.ShardIdx + << ": pathId: " << shardInfo.PathId + << ", at schemeshard: " << Self->TabletID()); + return false; + } + + const auto& settings = tableInfo->TTLSettings().GetEnabled(); + const TDuration expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds()); + const TInstant wallClock = ctx.Now() - expireAfter; + + NKikimrTxDataShard::TEvConditionalEraseRowsRequest request; + request.SetTableId(shardInfo.PathId.LocalPathId); + request.SetSchemaVersion(tableInfo->AlterVersion); + + for (const auto& [indexId, columnIds]: MakeIndexes(shardInfo.PathId)) { + auto& index = *request.MutableIndexes()->Add(); + + index.SetOwnerId(indexId.PathId.OwnerId); + index.SetPathId(indexId.PathId.LocalPathId); + index.SetSchemaVersion(indexId.SchemaVersion); + + for (const auto& [indexColumnId, mainColumnId] : columnIds) { + auto& keyMap = *index.MutableKeyMap()->Add(); + keyMap.SetIndexColumnId(indexColumnId); + keyMap.SetMainColumnId(mainColumnId); + } + } + + request.MutableExpiration()->SetColumnId(tableInfo->GetTTLColumnId()); + request.MutableExpiration()->SetWallClockTimestamp(wallClock.GetValue()); + request.MutableExpiration()->SetColumnUnit(settings.GetColumnUnit()); + + const auto& sysSettings = settings.GetSysSettings(); + request.MutableLimits()->SetBatchMaxBytes(sysSettings.GetBatchMaxBytes()); + request.MutableLimits()->SetBatchMinKeys(sysSettings.GetBatchMinKeys()); + request.MutableLimits()->SetBatchMaxKeys(sysSettings.GetBatchMaxKeys()); + + RunOnTablets.emplace(shardInfo.TabletID, std::move(request)); + + return true; + } + + void DoComplete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxRunConditionalErase DoComplete" + << ": at schemeshard: " << Self->TabletID()); + + for (auto& kv : RunOnTablets) { + const auto& tabletId = kv.first; + auto& request = kv.second; + + auto ev = MakeHolder<TEvDataShard::TEvConditionalEraseRowsRequest>(); + ev->Record = std::move(request); + + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Run conditional erase" + << ", tabletId: " << tabletId + << ", request: " << ev->Record.ShortDebugString() + << ", at schemeshard: " << Self->TabletID()); + Self->PipeClientCache->Send(ctx, ui64(tabletId), ev.Release()); + } + + if (!TableInfo) { + Self->ScheduleConditionalEraseRun(ctx); + } + } + +private: + THashMap<TTableId, TVector<std::pair<ui32, ui32>>> MakeIndexes(const TPathId& mainPathId) const { + THashMap<TTableId, TVector<std::pair<ui32, ui32>>> result; + + Y_VERIFY(Self->PathsById.contains(mainPathId)); + auto mainPath = Self->PathsById.at(mainPathId); + + Y_VERIFY(Self->Tables.contains(mainPathId)); + auto mainTable = Self->Tables.at(mainPathId); + + for (const auto& [_, childPathId] : mainPath->GetChildren()) { + Y_VERIFY(Self->PathsById.contains(childPathId)); + auto childPath = Self->PathsById.at(childPathId); + + if (!childPath->IsTableIndex() || childPath->Dropped()) { + continue; + } + + auto index = GetIndex(childPath); + if (index->Type == NKikimrSchemeOp::EIndexTypeGlobalAsync) { + continue; + } + + auto [indexImplTableId, indexImplTable] = GetIndexImplTable(childPath); + + auto ret = result.emplace( + TTableId(indexImplTableId, indexImplTable->AlterVersion), + MakeColumnIds(mainTable, index, indexImplTable) + ); + Y_VERIFY(ret.second); + } + + return result; + } + + TTableIndexInfo::TPtr GetIndex(TPathElement::TPtr indexPath) const { + Y_VERIFY(Self->Indexes.contains(indexPath->PathId)); + return Self->Indexes.at(indexPath->PathId); + } + + std::pair<TPathId, TTableInfo::TPtr> GetIndexImplTable(TPathElement::TPtr indexPath) const { + Y_VERIFY(indexPath->GetChildren().size() == 1); + + for (const auto& [_, indexImplPathId] : indexPath->GetChildren()) { auto childPath = Self->PathsById.at(indexImplPathId); Y_VERIFY(!childPath->Dropped()); - Y_VERIFY(Self->Tables.contains(indexImplPathId)); - return std::make_pair(indexImplPathId, Self->Tables.at(indexImplPathId)); - } - - Y_FAIL("Unreachable"); - } - - static TVector<std::pair<ui32, ui32>> MakeColumnIds(TTableInfo::TPtr mainTable, TTableIndexInfo::TPtr index, TTableInfo::TPtr indexImplTable) { - TVector<std::pair<ui32, ui32>> result; - THashSet<TString> keys; - - const auto mainColumns = MakeColumnNameToId(mainTable->Columns); - const auto indexImplColumns = MakeColumnNameToId(indexImplTable->Columns); - - for (const TString& indexKey : index->IndexKeys) { - Y_VERIFY(mainColumns.contains(indexKey)); - Y_VERIFY(indexImplColumns.contains(indexKey)); - - result.emplace_back(std::make_pair(indexImplColumns.at(indexKey), mainColumns.at(indexKey))); - keys.insert(indexKey); - } - - for (const ui32 mainColumnId : mainTable->KeyColumnIds) { - Y_VERIFY(mainTable->Columns.contains(mainColumnId)); - const TString& mainKey = mainTable->Columns.at(mainColumnId).Name; - - if (keys.contains(mainKey)) { - continue; - } - - Y_VERIFY(indexImplColumns.contains(mainKey)); - result.emplace_back(std::make_pair(indexImplColumns.at(mainKey), mainColumnId)); - } - - return result; - } - - static THashMap<TString, ui32> MakeColumnNameToId(const THashMap<ui32, TTableInfo::TColumn>& columns) { - THashMap<TString, ui32> result; - - for (const auto& [id, column] : columns) { - if (column.IsDropped()) { - continue; - } - - auto ret = result.emplace(column.Name, id); - Y_VERIFY(ret.second); - } - - return result; - } - -}; // TTxRunConditionalErase - + Y_VERIFY(Self->Tables.contains(indexImplPathId)); + return std::make_pair(indexImplPathId, Self->Tables.at(indexImplPathId)); + } + + Y_FAIL("Unreachable"); + } + + static TVector<std::pair<ui32, ui32>> MakeColumnIds(TTableInfo::TPtr mainTable, TTableIndexInfo::TPtr index, TTableInfo::TPtr indexImplTable) { + TVector<std::pair<ui32, ui32>> result; + THashSet<TString> keys; + + const auto mainColumns = MakeColumnNameToId(mainTable->Columns); + const auto indexImplColumns = MakeColumnNameToId(indexImplTable->Columns); + + for (const TString& indexKey : index->IndexKeys) { + Y_VERIFY(mainColumns.contains(indexKey)); + Y_VERIFY(indexImplColumns.contains(indexKey)); + + result.emplace_back(std::make_pair(indexImplColumns.at(indexKey), mainColumns.at(indexKey))); + keys.insert(indexKey); + } + + for (const ui32 mainColumnId : mainTable->KeyColumnIds) { + Y_VERIFY(mainTable->Columns.contains(mainColumnId)); + const TString& mainKey = mainTable->Columns.at(mainColumnId).Name; + + if (keys.contains(mainKey)) { + continue; + } + + Y_VERIFY(indexImplColumns.contains(mainKey)); + result.emplace_back(std::make_pair(indexImplColumns.at(mainKey), mainColumnId)); + } + + return result; + } + + static THashMap<TString, ui32> MakeColumnNameToId(const THashMap<ui32, TTableInfo::TColumn>& columns) { + THashMap<TString, ui32> result; + + for (const auto& [id, column] : columns) { + if (column.IsDropped()) { + continue; + } + + auto ret = result.emplace(column.Name, id); + Y_VERIFY(ret.second); + } + + return result; + } + +}; // TTxRunConditionalErase + struct TSchemeShard::TTxScheduleConditionalErase : public TTransactionBase<TSchemeShard> { - TEvDataShard::TEvConditionalEraseRowsResponse::TPtr Ev; - THolder<NSysView::TEvSysView::TEvUpdateTtlStats> StatsCollectorEv; - TTableInfo::TPtr TableInfo; - - TTxScheduleConditionalErase(TSelf* self, TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev) - : TBase(self) - , Ev(ev) - , TableInfo(nullptr) - { - } - - virtual ~TTxScheduleConditionalErase() = default; - - TTxType GetTxType() const override { - return TXTYPE_SCHEDULE_CONDITIONAL_ERASE; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxScheduleConditionalErase Execute" - << ": at schemeshard: " << Self->TabletID()); - - if (!Self->AllowConditionalEraseOperations) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase operations are not allowed" - << ", skip TTxScheduleConditionalErase" - << ": at schemeshard: " << Self->TabletID()); - return true; - } - - const auto& record = Ev->Get()->Record; - - const TTabletId tabletId(record.GetTabletID()); - const TShardIdx shardIdx = Self->GetShardIdx(tabletId); - - if (!Self->ShardInfos.contains(shardIdx)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" - << ": tabletId: " << tabletId - << ", at schemeshard: " << Self->TabletID()); - return true; - } - - const TShardInfo& shardInfo = Self->ShardInfos.at(shardIdx); - const TPathId& tableId = shardInfo.PathId; - - if (!Self->TTLEnabledTables.contains(tableId)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P2" - << ": tabletId: " << tabletId - << ", tableId: " << tableId - << ", at schemeshard: " << Self->TabletID()); - return true; - } - - TTableInfo::TPtr tableInfo = Self->TTLEnabledTables.at(tableId); - if (!tableInfo->IsTTLEnabled()) { - LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P3" - << ": tabletId: " << tabletId - << ", tableId: " << tableId - << ", at schemeshard: " << Self->TabletID()); - return true; - } - - const auto& inFlight = tableInfo->GetInFlightCondErase(); - if (!inFlight.contains(shardIdx)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Shard idx mismatch" - << ", tableId: " << tableId - << ", shardIdx: " << shardIdx - << ", in-flight shards: [" << JoinSeq(",", inFlight) << "]" - << ", at schemeshard: " << Self->TabletID()); - return true; - } - - const auto& sysSettings = tableInfo->TTLSettings().GetEnabled().GetSysSettings(); - TDuration next = TDuration::FromValue(sysSettings.GetRunInterval()); - - switch (record.GetStatus()) { - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::PARTIAL: - // TODO: remember progress - return true; - - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::BAD_REQUEST: - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ABORTED: - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ERASE_ERROR: - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OVERLOADED: - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::SCHEME_ERROR: - next = TDuration::FromValue(sysSettings.GetRetryInterval()); - LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unsuccessful conditional erase" - << ": tabletId: " << tabletId - << ", status: " << NKikimrTxDataShard::TEvConditionalEraseRowsResponse_EStatus_Name(record.GetStatus()) - << ", error: " << record.GetErrorDescription() - << ", retry after: " << next - << ", at schemeshard: " << Self->TabletID()); - break; - - case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OK: - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Successful conditional erase" - << ": tabletId: " << tabletId - << ", at schemeshard: " << Self->TabletID()); - break; - - default: - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unknown conditional erase status" - << ": tabletId: " << tabletId - << ", status: " << static_cast<ui32>(record.GetStatus()) - << ", error: " << record.GetErrorDescription() - << ", at schemeshard: " << Self->TabletID()); - break; - } - - const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); - Y_VERIFY(shardToPartition.contains(shardIdx)); - const ui64 partitionIdx = shardToPartition.at(shardIdx); - - const auto& partitions = tableInfo->GetPartitions(); - Y_VERIFY(partitionIdx < partitions.size()); - const auto& tableShardInfo = partitions.at(partitionIdx); - - const auto& lag = tableShardInfo.LastCondEraseLag; - if (lag) { - Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); - } - - const auto now = ctx.Now(); - - NIceDb::TNiceDb db(txc.DB); - tableInfo->ScheduleNextCondErase(shardIdx, now, next); - Self->PersistTablePartitionCondErase(db, tableId, partitionIdx, tableInfo); - - if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) { - StatsCollectorEv = MakeHolder<NSysView::TEvSysView::TEvUpdateTtlStats>( - Self->GetDomainKey(tableId), tableId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId())) - ); - - auto& stats = StatsCollectorEv->Stats; - stats.SetLastRunTime(now.MilliSeconds()); - stats.SetLastRowsProcessed(record.GetStats().GetRowsProcessed()); - stats.SetLastRowsErased(record.GetStats().GetRowsErased()); - } - - Y_VERIFY(lag.Defined()); - Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); - - TableInfo = tableInfo; - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxScheduleConditionalErase Complete" - << ": at schemeshard: " << Self->TabletID()); - - if (StatsCollectorEv) { - ctx.Send(Self->SysPartitionStatsCollector, StatsCollectorEv.Release()); - } - - if (TableInfo) { - Self->Execute(new TTxRunConditionalErase(Self, TableInfo), ctx); - } - } - -}; // TTxScheduleConditionalErase - + TEvDataShard::TEvConditionalEraseRowsResponse::TPtr Ev; + THolder<NSysView::TEvSysView::TEvUpdateTtlStats> StatsCollectorEv; + TTableInfo::TPtr TableInfo; + + TTxScheduleConditionalErase(TSelf* self, TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev) + : TBase(self) + , Ev(ev) + , TableInfo(nullptr) + { + } + + virtual ~TTxScheduleConditionalErase() = default; + + TTxType GetTxType() const override { + return TXTYPE_SCHEDULE_CONDITIONAL_ERASE; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxScheduleConditionalErase Execute" + << ": at schemeshard: " << Self->TabletID()); + + if (!Self->AllowConditionalEraseOperations) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase operations are not allowed" + << ", skip TTxScheduleConditionalErase" + << ": at schemeshard: " << Self->TabletID()); + return true; + } + + const auto& record = Ev->Get()->Record; + + const TTabletId tabletId(record.GetTabletID()); + const TShardIdx shardIdx = Self->GetShardIdx(tabletId); + + if (!Self->ShardInfos.contains(shardIdx)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" + << ": tabletId: " << tabletId + << ", at schemeshard: " << Self->TabletID()); + return true; + } + + const TShardInfo& shardInfo = Self->ShardInfos.at(shardIdx); + const TPathId& tableId = shardInfo.PathId; + + if (!Self->TTLEnabledTables.contains(tableId)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P2" + << ": tabletId: " << tabletId + << ", tableId: " << tableId + << ", at schemeshard: " << Self->TabletID()); + return true; + } + + TTableInfo::TPtr tableInfo = Self->TTLEnabledTables.at(tableId); + if (!tableInfo->IsTTLEnabled()) { + LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTL is not enabled for table #P3" + << ": tabletId: " << tabletId + << ", tableId: " << tableId + << ", at schemeshard: " << Self->TabletID()); + return true; + } + + const auto& inFlight = tableInfo->GetInFlightCondErase(); + if (!inFlight.contains(shardIdx)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Shard idx mismatch" + << ", tableId: " << tableId + << ", shardIdx: " << shardIdx + << ", in-flight shards: [" << JoinSeq(",", inFlight) << "]" + << ", at schemeshard: " << Self->TabletID()); + return true; + } + + const auto& sysSettings = tableInfo->TTLSettings().GetEnabled().GetSysSettings(); + TDuration next = TDuration::FromValue(sysSettings.GetRunInterval()); + + switch (record.GetStatus()) { + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::PARTIAL: + // TODO: remember progress + return true; + + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::BAD_REQUEST: + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ABORTED: + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ERASE_ERROR: + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OVERLOADED: + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::SCHEME_ERROR: + next = TDuration::FromValue(sysSettings.GetRetryInterval()); + LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unsuccessful conditional erase" + << ": tabletId: " << tabletId + << ", status: " << NKikimrTxDataShard::TEvConditionalEraseRowsResponse_EStatus_Name(record.GetStatus()) + << ", error: " << record.GetErrorDescription() + << ", retry after: " << next + << ", at schemeshard: " << Self->TabletID()); + break; + + case NKikimrTxDataShard::TEvConditionalEraseRowsResponse::OK: + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Successful conditional erase" + << ": tabletId: " << tabletId + << ", at schemeshard: " << Self->TabletID()); + break; + + default: + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unknown conditional erase status" + << ": tabletId: " << tabletId + << ", status: " << static_cast<ui32>(record.GetStatus()) + << ", error: " << record.GetErrorDescription() + << ", at schemeshard: " << Self->TabletID()); + break; + } + + const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); + Y_VERIFY(shardToPartition.contains(shardIdx)); + const ui64 partitionIdx = shardToPartition.at(shardIdx); + + const auto& partitions = tableInfo->GetPartitions(); + Y_VERIFY(partitionIdx < partitions.size()); + const auto& tableShardInfo = partitions.at(partitionIdx); + + const auto& lag = tableShardInfo.LastCondEraseLag; + if (lag) { + Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); + } + + const auto now = ctx.Now(); + + NIceDb::TNiceDb db(txc.DB); + tableInfo->ScheduleNextCondErase(shardIdx, now, next); + Self->PersistTablePartitionCondErase(db, tableId, partitionIdx, tableInfo); + + if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) { + StatsCollectorEv = MakeHolder<NSysView::TEvSysView::TEvUpdateTtlStats>( + Self->GetDomainKey(tableId), tableId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId())) + ); + + auto& stats = StatsCollectorEv->Stats; + stats.SetLastRunTime(now.MilliSeconds()); + stats.SetLastRowsProcessed(record.GetStats().GetRowsProcessed()); + stats.SetLastRowsErased(record.GetStats().GetRowsErased()); + } + + Y_VERIFY(lag.Defined()); + Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); + + TableInfo = tableInfo; + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxScheduleConditionalErase Complete" + << ": at schemeshard: " << Self->TabletID()); + + if (StatsCollectorEv) { + ctx.Send(Self->SysPartitionStatsCollector, StatsCollectorEv.Release()); + } + + if (TableInfo) { + Self->Execute(new TTxRunConditionalErase(Self, TableInfo), ctx); + } + } + +}; // TTxScheduleConditionalErase + ITransaction* TSchemeShard::CreateTxRunConditionalErase(TEvPrivate::TEvRunConditionalErase::TPtr& ev) { - return new TTxRunConditionalErase(this, ev); -} - + return new TTxRunConditionalErase(this, ev); +} + ITransaction* TSchemeShard::CreateTxScheduleConditionalErase(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev) { - return new TTxScheduleConditionalErase(this, ev); -} - + return new TTxScheduleConditionalErase(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp index 7e2f3d0c7b1..855553cddc8 100644 --- a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp @@ -98,9 +98,9 @@ struct TSchemeShard::TTxDeleteTabletReply : public TSchemeShard::TRwTxBase { case ETabletType::SequenceShard: Self->TabletCounters->Simple()[COUNTER_SEQUENCESHARD_COUNT].Sub(1); break; - case ETabletType::ReplicationController: - Self->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Sub(1); - break; + case ETabletType::ReplicationController: + Self->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Sub(1); + break; default: Y_FAIL_S("Unknown TabletType" << ", ShardIdx " << ShardIdx @@ -124,15 +124,15 @@ struct TSchemeShard::TTxDeleteTabletReply : public TSchemeShard::TRwTxBase { auto domain = Self->ResolveDomainInfo(path); domain->RemoveInternalShard(ShardIdx); - switch (tabletType) { - case ETabletType::SequenceShard: + switch (tabletType) { + case ETabletType::SequenceShard: domain->RemoveSequenceShard(ShardIdx); - break; - case ETabletType::ReplicationController: - domain->RemoveReplicationController(ShardIdx); - break; - default: - break; + break; + case ETabletType::ReplicationController: + domain->RemoveReplicationController(ShardIdx); + break; + default: + break; } TabletId = shardInfo.TabletID; diff --git a/ydb/core/tx/schemeshard/schemeshard__describe_scheme.cpp b/ydb/core/tx/schemeshard/schemeshard__describe_scheme.cpp index da41a3f0848..e6d96356527 100644 --- a/ydb/core/tx/schemeshard/schemeshard__describe_scheme.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__describe_scheme.cpp @@ -1,5 +1,5 @@ #include "schemeshard_impl.h" -#include "schemeshard_path_describer.h" +#include "schemeshard_path_describer.h" #include <util/stream/format.h> @@ -10,16 +10,16 @@ using namespace NTabletFlatExecutor; struct TSchemeShard::TTxDescribeScheme : public TSchemeShard::TRwTxBase { const TActorId Sender; - const ui64 Cookie; - TPathDescriber PathDescriber; + const ui64 Cookie; + TPathDescriber PathDescriber; THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> Result; - + TTxDescribeScheme(TSelf *self, TEvSchemeShard::TEvDescribeScheme::TPtr &ev) : TRwTxBase(self) - , Sender(ev->Sender) - , Cookie(ev->Cookie) - , PathDescriber(self, std::move(ev->Get()->Record)) + , Sender(ev->Sender) + , Cookie(ev->Cookie) + , PathDescriber(self, std::move(ev->Get()->Record)) {} TTxType GetTxType() const override { return TXTYPE_DESCRIBE_SCHEME; } @@ -27,25 +27,25 @@ struct TSchemeShard::TTxDescribeScheme : public TSchemeShard::TRwTxBase { void DoExecute(TTransactionContext& /*txc*/, const TActorContext& ctx) override { LOG_DEBUG_S(ctx, NKikimrServices::SCHEMESHARD_DESCRIBE, "TTxDescribeScheme DoExecute" - << ", record: " << PathDescriber.GetParams().ShortDebugString() + << ", record: " << PathDescriber.GetParams().ShortDebugString() << ", at schemeshard: " << Self->TabletID()); - Result = PathDescriber.Describe(ctx); + Result = PathDescriber.Describe(ctx); } void DoComplete(const TActorContext &ctx) override { - const auto& params = PathDescriber.GetParams(); - - if (params.HasPathId()) { + const auto& params = PathDescriber.GetParams(); + + if (params.HasPathId()) { LOG_INFO_S(ctx, NKikimrServices::SCHEMESHARD_DESCRIBE, "Tablet " << Self->TabletID() - << " describe pathId " << params.GetPathId() + << " describe pathId " << params.GetPathId() << " took " << HumanReadable(ExecuteDuration) << " result status " <<NKikimrScheme::EStatus_Name(Result->Record.GetStatus())); } else { LOG_INFO_S(ctx, NKikimrServices::SCHEMESHARD_DESCRIBE, "Tablet " << Self->TabletID() - << " describe path \"" << params.GetPath() << "\"" + << " describe path \"" << params.GetPath() << "\"" << " took " << HumanReadable(ExecuteDuration) << " result status " <<NKikimrScheme::EStatus_Name(Result->Record.GetStatus())); } @@ -55,7 +55,7 @@ struct TSchemeShard::TTxDescribeScheme : public TSchemeShard::TRwTxBase { << ", result: " << Result->GetRecord().ShortDebugString() << ", at schemeshard: " << Self->TabletID()); - ctx.Send(Sender, std::move(Result), 0, Cookie); + ctx.Send(Sender, std::move(Result), 0, Cookie); } }; diff --git a/ydb/core/tx/schemeshard/schemeshard__fix_bad_paths.cpp b/ydb/core/tx/schemeshard/schemeshard__fix_bad_paths.cpp index c8d057c019c..b1755db0c1b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__fix_bad_paths.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__fix_bad_paths.cpp @@ -1,59 +1,59 @@ -#include "schemeshard_impl.h" - -#if defined SS_LOG_W -#error log macro redefinition -#endif - -#define SS_LOG_W(ctx, stream) LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, stream) - -namespace NKikimr { +#include "schemeshard_impl.h" + +#if defined SS_LOG_W +#error log macro redefinition +#endif + +#define SS_LOG_W(ctx, stream) LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, stream) + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TTxFixBadPaths : public TTransactionBase<TSchemeShard> { - explicit TTxFixBadPaths(TSelf *self) - : TBase(self) - { - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - NIceDb::TNiceDb db(txc.DB); - - for (auto& el : Self->PathsById) { - TPathId pathId = el.first; - TPathElement::TPtr pathEl = el.second; - - if (pathEl->Dropped() || !pathEl->NormalState()) { - continue; - } - + explicit TTxFixBadPaths(TSelf *self) + : TBase(self) + { + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + NIceDb::TNiceDb db(txc.DB); + + for (auto& el : Self->PathsById) { + TPathId pathId = el.first; + TPathElement::TPtr pathEl = el.second; + + if (pathEl->Dropped() || !pathEl->NormalState()) { + continue; + } + if (pathEl->CreateTxId == TTxId(0)) { pathEl->CreateTxId = TTxId(1); - Self->PersistCreateTxId(db, pathId, pathEl->CreateTxId); - - SS_LOG_W(ctx, "Fix CreateTxId, self# " << Self->TabletID() << ", pathId# " << pathId); - } + Self->PersistCreateTxId(db, pathId, pathEl->CreateTxId); + + SS_LOG_W(ctx, "Fix CreateTxId, self# " << Self->TabletID() << ", pathId# " << pathId); + } if (pathId != Self->RootPathId() && pathEl->StepCreated == InvalidStepId) { pathEl->StepCreated = TStepId(1); - Self->PersistCreateStep(db, pathId, pathEl->StepCreated); - - SS_LOG_W(ctx, "Fix StepCreated, self# " << Self->TabletID() << ", pathId# " << pathId); - } - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - Self->Execute(Self->CreateTxInitPopulator(TSideEffects::TPublications()), ctx); - } - + Self->PersistCreateStep(db, pathId, pathEl->StepCreated); + + SS_LOG_W(ctx, "Fix StepCreated, self# " << Self->TabletID() << ", pathId# " << pathId); + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + Self->Execute(Self->CreateTxInitPopulator(TSideEffects::TPublications()), ctx); + } + }; // TSchemeShard::TTxFixBadPaths - + NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxFixBadPaths() { - return new TTxFixBadPaths(this); -} - + return new TTxFixBadPaths(this); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 0521ececdda..650506549e0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -13,11 +13,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TSideEffects OnComplete; TMemoryChanges MemChanges; TStorageChanges DbChanges; - THashMap<TTxId, TDeque<TPathId>> Publications; + THashMap<TTxId, TDeque<TPathId>> Publications; TVector<TPathId> TablesToClean; TDeque<TPathId> BlockStoreVolumesToClean; - TVector<ui64> ExportsToResume; - TVector<ui64> ImportsToResume; + TVector<ui64> ExportsToResume; + TVector<ui64> ImportsToResume; explicit TTxInit(TSelf *self) : TBase(self) @@ -313,7 +313,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui32, ui64, TString, TString, TString, ui64, TString, bool> TTableRec; + typedef std::tuple<TPathId, ui32, ui64, TString, TString, TString, ui64, TString, bool> TTableRec; typedef TDeque<TTableRec> TTableRows; bool LoadTables(NIceDb::TNiceDb& db, TTableRows& tableRows) const { @@ -331,11 +331,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TString alterTabletFull = rowSet.GetValueOrDefault<Schema::Tables::AlterTableFull>(); TString alterTabletDiff = rowSet.GetValueOrDefault<Schema::Tables::AlterTable>(); ui64 partitionVersion = rowSet.GetValueOrDefault<Schema::Tables::PartitioningVersion>(0); - TString ttlSettings = rowSet.GetValueOrDefault<Schema::Tables::TTLSettings>(); - bool isBackup = rowSet.GetValueOrDefault<Schema::Tables::IsBackup>(false); + TString ttlSettings = rowSet.GetValueOrDefault<Schema::Tables::TTLSettings>(); + bool isBackup = rowSet.GetValueOrDefault<Schema::Tables::IsBackup>(false); tableRows.emplace_back(pathId, - nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); + nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); if (!rowSet.Next()) { return false; @@ -359,11 +359,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TString alterTabletFull = rowSet.GetValueOrDefault<Schema::MigratedTables::AlterTableFull>(); TString alterTabletDiff = rowSet.GetValueOrDefault<Schema::MigratedTables::AlterTable>(); ui64 partitionVersion = rowSet.GetValueOrDefault<Schema::MigratedTables::PartitioningVersion>(0); - TString ttlSettings = rowSet.GetValueOrDefault<Schema::MigratedTables::TTLSettings>(); - bool isBackup = rowSet.GetValueOrDefault<Schema::MigratedTables::IsBackup>(false); + TString ttlSettings = rowSet.GetValueOrDefault<Schema::MigratedTables::TTLSettings>(); + bool isBackup = rowSet.GetValueOrDefault<Schema::MigratedTables::IsBackup>(false); tableRows.emplace_back(pathId, - nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); + nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); if (!rowSet.Next()) { return false; @@ -509,7 +509,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui64, TString, TShardIdx, ui64, ui64> TTablePartitionRec; + typedef std::tuple<TPathId, ui64, TString, TShardIdx, ui64, ui64> TTablePartitionRec; typedef TDeque<TTablePartitionRec> TTablePartitionsRows; bool LoadTablePartitions(NIceDb::TNiceDb& db, TTablePartitionsRows& partitionsRows) const { @@ -524,10 +524,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui64 id = rowSet.GetValue<Schema::TablePartitions::Id>(); TString rangeEnd = rowSet.GetValue<Schema::TablePartitions::RangeEnd>(); TShardIdx datashardIdx = Self->MakeLocalId(rowSet.GetValue<Schema::TablePartitions::DatashardIdx>()); - ui64 lastCondErase = rowSet.GetValueOrDefault<Schema::TablePartitions::LastCondErase>(0); - ui64 nextCondErase = rowSet.GetValueOrDefault<Schema::TablePartitions::NextCondErase>(0); + ui64 lastCondErase = rowSet.GetValueOrDefault<Schema::TablePartitions::LastCondErase>(0); + ui64 nextCondErase = rowSet.GetValueOrDefault<Schema::TablePartitions::NextCondErase>(0); - partitionsRows.emplace_back(pathId, id, rangeEnd, datashardIdx, lastCondErase, nextCondErase); + partitionsRows.emplace_back(pathId, id, rangeEnd, datashardIdx, lastCondErase, nextCondErase); if (!rowSet.Next()) { return false; @@ -549,10 +549,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TString rangeEnd = rowSet.GetValue<Schema::MigratedTablePartitions::RangeEnd>(); TShardIdx datashardIdx = TShardIdx(rowSet.GetValue<Schema::MigratedTablePartitions::OwnerShardIdx>(), rowSet.GetValue<Schema::MigratedTablePartitions::LocalShardIdx>()); - ui64 lastCondErase = rowSet.GetValueOrDefault<Schema::MigratedTablePartitions::LastCondErase>(0); - ui64 nextCondErase = rowSet.GetValueOrDefault<Schema::MigratedTablePartitions::NextCondErase>(0); + ui64 lastCondErase = rowSet.GetValueOrDefault<Schema::MigratedTablePartitions::LastCondErase>(0); + ui64 nextCondErase = rowSet.GetValueOrDefault<Schema::MigratedTablePartitions::NextCondErase>(0); - partitionsRows.emplace_back(pathId, id, rangeEnd, datashardIdx, lastCondErase, nextCondErase); + partitionsRows.emplace_back(pathId, id, rangeEnd, datashardIdx, lastCondErase, nextCondErase); if (!rowSet.Next()) { return false; @@ -820,7 +820,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, TString, TString, TString, TString, bool, TString, ui32> TBackupSettingsRec; + typedef std::tuple<TPathId, TString, TString, TString, TString, bool, TString, ui32> TBackupSettingsRec; typedef TDeque<TBackupSettingsRec> TBackupSettingsRows; bool LoadBackupSettings(NIceDb::TNiceDb& db, TBackupSettingsRows& settings) const { @@ -833,14 +833,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { while (!rowSet.EndOfSet()) { auto pathId = Self->MakeLocalId(rowSet.GetValue<Schema::BackupSettings::PathId>()); auto tableName = rowSet.GetValue<Schema::BackupSettings::TableName>(); - auto ytSettings = rowSet.GetValueOrDefault<Schema::BackupSettings::YTSettings>(""); - auto s3Settings = rowSet.GetValueOrDefault<Schema::BackupSettings::S3Settings>(""); - auto scanSettings = rowSet.GetValueOrDefault<Schema::BackupSettings::ScanSettings>(""); - auto needToBill = rowSet.GetValueOrDefault<Schema::BackupSettings::NeedToBill>(true); - auto tableDesc = rowSet.GetValueOrDefault<Schema::BackupSettings::TableDescription>(""); - auto nRetries = rowSet.GetValueOrDefault<Schema::BackupSettings::NumberOfRetries>(0); + auto ytSettings = rowSet.GetValueOrDefault<Schema::BackupSettings::YTSettings>(""); + auto s3Settings = rowSet.GetValueOrDefault<Schema::BackupSettings::S3Settings>(""); + auto scanSettings = rowSet.GetValueOrDefault<Schema::BackupSettings::ScanSettings>(""); + auto needToBill = rowSet.GetValueOrDefault<Schema::BackupSettings::NeedToBill>(true); + auto tableDesc = rowSet.GetValueOrDefault<Schema::BackupSettings::TableDescription>(""); + auto nRetries = rowSet.GetValueOrDefault<Schema::BackupSettings::NumberOfRetries>(0); - settings.emplace_back(pathId, tableName, ytSettings, s3Settings, scanSettings, needToBill, tableDesc, nRetries); + settings.emplace_back(pathId, tableName, ytSettings, s3Settings, scanSettings, needToBill, tableDesc, nRetries); if (!rowSet.Next()) { return false; @@ -858,14 +858,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto pathId = TPathId(rowSet.GetValue<Schema::MigratedBackupSettings::OwnerPathId>(), rowSet.GetValue<Schema::MigratedBackupSettings::LocalPathId>()); auto tableName = rowSet.GetValue<Schema::MigratedBackupSettings::TableName>(); - auto ytSettings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::YTSettings>(""); - auto s3Settings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::S3Settings>(""); - auto scanSettings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::ScanSettings>(""); - auto needToBill = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::NeedToBill>(true); - auto tableDesc = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::TableDescription>(""); - auto nRetries = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::NumberOfRetries>(0); + auto ytSettings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::YTSettings>(""); + auto s3Settings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::S3Settings>(""); + auto scanSettings = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::ScanSettings>(""); + auto needToBill = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::NeedToBill>(true); + auto tableDesc = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::TableDescription>(""); + auto nRetries = rowSet.GetValueOrDefault<Schema::MigratedBackupSettings::NumberOfRetries>(0); - settings.emplace_back(pathId, tableName, ytSettings, s3Settings, scanSettings, needToBill, tableDesc, nRetries); + settings.emplace_back(pathId, tableName, ytSettings, s3Settings, scanSettings, needToBill, tableDesc, nRetries); if (!rowSet.Next()) { return false; @@ -876,10 +876,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, TTxId, ui64, ui32, ui32, ui64, ui64, ui8> TCompletedBackupRestoreRec; - typedef TDeque<TCompletedBackupRestoreRec> TCompletedBackupRestoreRows; + typedef std::tuple<TPathId, TTxId, ui64, ui32, ui32, ui64, ui64, ui8> TCompletedBackupRestoreRec; + typedef TDeque<TCompletedBackupRestoreRec> TCompletedBackupRestoreRows; - bool LoadBackupRestoreHistory(NIceDb::TNiceDb& db, TCompletedBackupRestoreRows& history) const { + bool LoadBackupRestoreHistory(NIceDb::TNiceDb& db, TCompletedBackupRestoreRows& history) const { { auto rowSet = db.Table<Schema::CompletedBackups>().Range().Select(); if (!rowSet.IsReady()) { @@ -895,9 +895,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto totalShardCount = rowSet.GetValue<Schema::CompletedBackups::TotalShardCount>(); auto startTime = rowSet.GetValue<Schema::CompletedBackups::StartTime>(); auto dataSize = rowSet.GetValue<Schema::CompletedBackups::DataTotalSize>(); - auto kind = rowSet.GetValueOrDefault<Schema::CompletedBackups::Kind>(0); + auto kind = rowSet.GetValueOrDefault<Schema::CompletedBackups::Kind>(0); - history.emplace_back(pathId, txId, completeTime, successShardsCount, totalShardCount, startTime, dataSize, kind); + history.emplace_back(pathId, txId, completeTime, successShardsCount, totalShardCount, startTime, dataSize, kind); if (!rowSet.Next()) { return false; @@ -921,9 +921,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto totalShardCount = rowSet.GetValue<Schema::MigratedCompletedBackups::TotalShardCount>(); auto startTime = rowSet.GetValue<Schema::MigratedCompletedBackups::StartTime>(); auto dataSize = rowSet.GetValue<Schema::MigratedCompletedBackups::DataTotalSize>(); - auto kind = rowSet.GetValueOrDefault<Schema::MigratedCompletedBackups::Kind>(0); + auto kind = rowSet.GetValueOrDefault<Schema::MigratedCompletedBackups::Kind>(0); - history.emplace_back(pathId, txId, completeTime, successShardsCount, totalShardCount, startTime, dataSize, kind); + history.emplace_back(pathId, txId, completeTime, successShardsCount, totalShardCount, startTime, dataSize, kind); if (!rowSet.Next()) { return false; @@ -934,24 +934,24 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TTxId, TShardIdx, bool, TString, ui64, ui64> TShardBackupStatusRec; + typedef std::tuple<TTxId, TShardIdx, bool, TString, ui64, ui64> TShardBackupStatusRec; typedef TDeque<TShardBackupStatusRec> TShardBackupStatusRows; - template <typename T, typename U, typename V> - bool LoadBackupStatusesImpl(TShardBackupStatusRows& statuses, - T& byShardBackupStatus, U& byMigratedShardBackupStatus, V& byTxShardStatus) const { + template <typename T, typename U, typename V> + bool LoadBackupStatusesImpl(TShardBackupStatusRows& statuses, + T& byShardBackupStatus, U& byMigratedShardBackupStatus, V& byTxShardStatus) const { { - T& rowSet = byShardBackupStatus; + T& rowSet = byShardBackupStatus; if (!rowSet.IsReady()) { return false; } while (!rowSet.EndOfSet()) { - auto txId = rowSet.template GetValue<Schema::ShardBackupStatus::TxId>(); - auto shardIdx = Self->MakeLocalId(rowSet.template GetValue<Schema::ShardBackupStatus::ShardIdx>()); - auto explain = rowSet.template GetValue<Schema::ShardBackupStatus::Explain>(); + auto txId = rowSet.template GetValue<Schema::ShardBackupStatus::TxId>(); + auto shardIdx = Self->MakeLocalId(rowSet.template GetValue<Schema::ShardBackupStatus::ShardIdx>()); + auto explain = rowSet.template GetValue<Schema::ShardBackupStatus::Explain>(); - statuses.emplace_back(txId, shardIdx, false, explain, 0, 0); + statuses.emplace_back(txId, shardIdx, false, explain, 0, 0); if (!rowSet.Next()) { return false; @@ -960,18 +960,43 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } { - U& rowSet = byMigratedShardBackupStatus; + U& rowSet = byMigratedShardBackupStatus; if (!rowSet.IsReady()) { return false; } while (!rowSet.EndOfSet()) { - auto txId = rowSet.template GetValue<Schema::MigratedShardBackupStatus::TxId>(); - auto shardIdx = TShardIdx(rowSet.template GetValue<Schema::MigratedShardBackupStatus::OwnerShardId>(), - rowSet.template GetValue<Schema::MigratedShardBackupStatus::LocalShardId>()); - auto explain = rowSet.template GetValue<Schema::MigratedShardBackupStatus::Explain>(); + auto txId = rowSet.template GetValue<Schema::MigratedShardBackupStatus::TxId>(); + auto shardIdx = TShardIdx(rowSet.template GetValue<Schema::MigratedShardBackupStatus::OwnerShardId>(), + rowSet.template GetValue<Schema::MigratedShardBackupStatus::LocalShardId>()); + auto explain = rowSet.template GetValue<Schema::MigratedShardBackupStatus::Explain>(); - statuses.emplace_back(txId, shardIdx, false, explain, 0, 0); + statuses.emplace_back(txId, shardIdx, false, explain, 0, 0); + + if (!rowSet.Next()) { + return false; + } + } + } + + { + V& rowSet = byTxShardStatus; + if (!rowSet.IsReady()) { + return false; + } + + while (!rowSet.EndOfSet()) { + auto txId = rowSet.template GetValue<Schema::TxShardStatus::TxId>(); + auto shardIdx = TShardIdx( + rowSet.template GetValue<Schema::TxShardStatus::OwnerShardId>(), + rowSet.template GetValue<Schema::TxShardStatus::LocalShardId>() + ); + auto success = rowSet.template GetValue<Schema::TxShardStatus::Success>(); + auto error = rowSet.template GetValue<Schema::TxShardStatus::Error>(); + auto bytes = rowSet.template GetValue<Schema::TxShardStatus::BytesProcessed>(); + auto rows = rowSet.template GetValue<Schema::TxShardStatus::RowsProcessed>(); + + statuses.emplace_back(txId, shardIdx, success, error, bytes, rows); if (!rowSet.Next()) { return false; @@ -979,42 +1004,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - { - V& rowSet = byTxShardStatus; - if (!rowSet.IsReady()) { - return false; - } - - while (!rowSet.EndOfSet()) { - auto txId = rowSet.template GetValue<Schema::TxShardStatus::TxId>(); - auto shardIdx = TShardIdx( - rowSet.template GetValue<Schema::TxShardStatus::OwnerShardId>(), - rowSet.template GetValue<Schema::TxShardStatus::LocalShardId>() - ); - auto success = rowSet.template GetValue<Schema::TxShardStatus::Success>(); - auto error = rowSet.template GetValue<Schema::TxShardStatus::Error>(); - auto bytes = rowSet.template GetValue<Schema::TxShardStatus::BytesProcessed>(); - auto rows = rowSet.template GetValue<Schema::TxShardStatus::RowsProcessed>(); - - statuses.emplace_back(txId, shardIdx, success, error, bytes, rows); - - if (!rowSet.Next()) { - return false; - } - } - } - return true; } - bool LoadBackupStatuses(NIceDb::TNiceDb& db, TShardBackupStatusRows& statuses) const { - auto byShardBackupStatus = db.Table<Schema::ShardBackupStatus>().Range().Select(); - auto byMigratedShardBackupStatus = db.Table<Schema::MigratedShardBackupStatus>().Range().Select(); - auto byTxShardStatus = db.Table<Schema::TxShardStatus>().Range().Select(); - - return LoadBackupStatusesImpl(statuses, byShardBackupStatus, byMigratedShardBackupStatus, byTxShardStatus); - } - + bool LoadBackupStatuses(NIceDb::TNiceDb& db, TShardBackupStatusRows& statuses) const { + auto byShardBackupStatus = db.Table<Schema::ShardBackupStatus>().Range().Select(); + auto byMigratedShardBackupStatus = db.Table<Schema::MigratedShardBackupStatus>().Range().Select(); + auto byTxShardStatus = db.Table<Schema::TxShardStatus>().Range().Select(); + + return LoadBackupStatusesImpl(statuses, byShardBackupStatus, byMigratedShardBackupStatus, byTxShardStatus); + } + typedef std::tuple<TPathId, ui64, NKikimrSchemeOp::EIndexType, NKikimrSchemeOp::EIndexState> TTableIndexRec; typedef TDeque<TTableIndexRec> TTableIndexRows; @@ -1453,94 +1453,94 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - // Read SubDomains + // Read SubDomains { TSchemeLimits rootLimits = TSchemeShard::DefaultLimits; - - if (Self->PathsById.contains(Self->RootPathId()) && Self->IsDomainSchemeShard) { - auto row = db.Table<Schema::SubDomains>().Key(Self->RootPathId().LocalPathId).Select(); - - if (!row.IsReady()) - return false; - - ui64 version = 0; - - if (row.IsValid()) { - version = row.GetValue<Schema::SubDomains::AlterVersion>(); - rootLimits.MaxDepth = row.GetValueOrDefault<Schema::SubDomains::DepthLimit>(rootLimits.MaxDepth); - rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPaths); - rootLimits.MaxChildrenInDir = row.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(rootLimits.MaxChildrenInDir); - rootLimits.MaxAclBytesSize = row.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(rootLimits.MaxAclBytesSize); - rootLimits.MaxTableColumns = row.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(rootLimits.MaxTableColumns); - rootLimits.MaxTableColumnNameLength = row.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(rootLimits.MaxTableColumnNameLength); - rootLimits.MaxTableKeyColumns = row.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(rootLimits.MaxTableKeyColumns); - rootLimits.MaxTableIndices = row.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(rootLimits.MaxTableIndices); - rootLimits.MaxShards = row.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(rootLimits.MaxShards); - rootLimits.MaxShardsInPath = row.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(rootLimits.MaxShardsInPath); - rootLimits.MaxConsistentCopyTargets = row.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(rootLimits.MaxConsistentCopyTargets); - rootLimits.MaxPathElementLength = row.GetValueOrDefault<Schema::SubDomains::PathElementLength>(rootLimits.MaxPathElementLength); - rootLimits.ExtraPathSymbolsAllowed = row.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(rootLimits.ExtraPathSymbolsAllowed); + + if (Self->PathsById.contains(Self->RootPathId()) && Self->IsDomainSchemeShard) { + auto row = db.Table<Schema::SubDomains>().Key(Self->RootPathId().LocalPathId).Select(); + + if (!row.IsReady()) + return false; + + ui64 version = 0; + + if (row.IsValid()) { + version = row.GetValue<Schema::SubDomains::AlterVersion>(); + rootLimits.MaxDepth = row.GetValueOrDefault<Schema::SubDomains::DepthLimit>(rootLimits.MaxDepth); + rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPaths); + rootLimits.MaxChildrenInDir = row.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(rootLimits.MaxChildrenInDir); + rootLimits.MaxAclBytesSize = row.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(rootLimits.MaxAclBytesSize); + rootLimits.MaxTableColumns = row.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(rootLimits.MaxTableColumns); + rootLimits.MaxTableColumnNameLength = row.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(rootLimits.MaxTableColumnNameLength); + rootLimits.MaxTableKeyColumns = row.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(rootLimits.MaxTableKeyColumns); + rootLimits.MaxTableIndices = row.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(rootLimits.MaxTableIndices); + rootLimits.MaxShards = row.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(rootLimits.MaxShards); + rootLimits.MaxShardsInPath = row.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(rootLimits.MaxShardsInPath); + rootLimits.MaxConsistentCopyTargets = row.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(rootLimits.MaxConsistentCopyTargets); + rootLimits.MaxPathElementLength = row.GetValueOrDefault<Schema::SubDomains::PathElementLength>(rootLimits.MaxPathElementLength); + rootLimits.ExtraPathSymbolsAllowed = row.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(rootLimits.ExtraPathSymbolsAllowed); rootLimits.MaxPQPartitions = row.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(rootLimits.MaxPQPartitions); - } - - TSubDomainInfo::TPtr rootDomainInfo = new TSubDomainInfo(version, Self->RootPathId()); - rootDomainInfo->SetSchemeLimits(rootLimits); + } + + TSubDomainInfo::TPtr rootDomainInfo = new TSubDomainInfo(version, Self->RootPathId()); + rootDomainInfo->SetSchemeLimits(rootLimits); rootDomainInfo->SetSecurityStateVersion(row.GetValueOrDefault<Schema::SubDomains::SecurityStateVersion>()); - - rootDomainInfo->InitializeAsGlobal(Self->CreateRootProcessingParams(ctx)); - - Self->SubDomains[Self->RootPathId()] = rootDomainInfo; - } - - auto rowset = db.Table<Schema::SubDomains>().Range().Select(); - if (!rowset.IsReady()) - return false; - while (!rowset.EndOfSet()) { - TLocalPathId localPathId = rowset.GetValue<Schema::SubDomains::PathId>(); - TPathId pathId(selfId, localPathId); - - Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); - - TPathElement::TPtr path = Self->PathsById.at(pathId); - Y_VERIFY_S(path->IsDomainRoot(), "Path is not a domain, pathId: " << pathId); - - if (!path->IsRoot() || !Self->IsDomainSchemeShard) { - Y_VERIFY(!Self->SubDomains.contains(pathId)); - - ui64 alterVersion = rowset.GetValue<Schema::SubDomains::AlterVersion>(); - ui64 planResolution = rowset.GetValue<Schema::SubDomains::PlanResolution>(); - ui32 timeCastBukets = rowset.GetValue<Schema::SubDomains::TimeCastBuckets>(); - TPathId resourcesDomainId = TPathId( - rowset.GetValue<Schema::SubDomains::ResourcesDomainOwnerPathId>(), - rowset.GetValue<Schema::SubDomains::ResourcesDomainLocalPathId>()); - TSubDomainInfo::TPtr domainInfo = new TSubDomainInfo( - alterVersion, - planResolution, - timeCastBukets, - resourcesDomainId); - Self->SubDomains[pathId] = domainInfo; + + rootDomainInfo->InitializeAsGlobal(Self->CreateRootProcessingParams(ctx)); + + Self->SubDomains[Self->RootPathId()] = rootDomainInfo; + } + + auto rowset = db.Table<Schema::SubDomains>().Range().Select(); + if (!rowset.IsReady()) + return false; + while (!rowset.EndOfSet()) { + TLocalPathId localPathId = rowset.GetValue<Schema::SubDomains::PathId>(); + TPathId pathId(selfId, localPathId); + + Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); + + TPathElement::TPtr path = Self->PathsById.at(pathId); + Y_VERIFY_S(path->IsDomainRoot(), "Path is not a domain, pathId: " << pathId); + + if (!path->IsRoot() || !Self->IsDomainSchemeShard) { + Y_VERIFY(!Self->SubDomains.contains(pathId)); + + ui64 alterVersion = rowset.GetValue<Schema::SubDomains::AlterVersion>(); + ui64 planResolution = rowset.GetValue<Schema::SubDomains::PlanResolution>(); + ui32 timeCastBukets = rowset.GetValue<Schema::SubDomains::TimeCastBuckets>(); + TPathId resourcesDomainId = TPathId( + rowset.GetValue<Schema::SubDomains::ResourcesDomainOwnerPathId>(), + rowset.GetValue<Schema::SubDomains::ResourcesDomainLocalPathId>()); + TSubDomainInfo::TPtr domainInfo = new TSubDomainInfo( + alterVersion, + planResolution, + timeCastBukets, + resourcesDomainId); + Self->SubDomains[pathId] = domainInfo; Self->IncrementPathDbRefCount(pathId); - - TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomains::SharedHiveId>(); - domainInfo->SetSharedHive(sharedHiveId); - - TSchemeLimits limits = rootLimits; - limits.MaxDepth = rowset.GetValueOrDefault<Schema::SubDomains::DepthLimit>(limits.MaxDepth); - limits.MaxPaths = rowset.GetValueOrDefault<Schema::SubDomains::PathsLimit>(limits.MaxPaths); - limits.MaxChildrenInDir = rowset.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(limits.MaxChildrenInDir); - limits.MaxAclBytesSize = rowset.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(limits.MaxAclBytesSize); - limits.MaxTableColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(limits.MaxTableColumns); - limits.MaxTableColumnNameLength = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(limits.MaxTableColumnNameLength); - limits.MaxTableKeyColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(limits.MaxTableKeyColumns); - limits.MaxTableIndices = rowset.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(limits.MaxTableIndices); - limits.MaxShards = rowset.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(limits.MaxShards); - limits.MaxShardsInPath = rowset.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(limits.MaxShardsInPath); - limits.MaxConsistentCopyTargets = rowset.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(limits.MaxConsistentCopyTargets); - limits.MaxPathElementLength = rowset.GetValueOrDefault<Schema::SubDomains::PathElementLength>(limits.MaxPathElementLength); - limits.ExtraPathSymbolsAllowed = rowset.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(limits.ExtraPathSymbolsAllowed); + + TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomains::SharedHiveId>(); + domainInfo->SetSharedHive(sharedHiveId); + + TSchemeLimits limits = rootLimits; + limits.MaxDepth = rowset.GetValueOrDefault<Schema::SubDomains::DepthLimit>(limits.MaxDepth); + limits.MaxPaths = rowset.GetValueOrDefault<Schema::SubDomains::PathsLimit>(limits.MaxPaths); + limits.MaxChildrenInDir = rowset.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(limits.MaxChildrenInDir); + limits.MaxAclBytesSize = rowset.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(limits.MaxAclBytesSize); + limits.MaxTableColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(limits.MaxTableColumns); + limits.MaxTableColumnNameLength = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(limits.MaxTableColumnNameLength); + limits.MaxTableKeyColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(limits.MaxTableKeyColumns); + limits.MaxTableIndices = rowset.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(limits.MaxTableIndices); + limits.MaxShards = rowset.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(limits.MaxShards); + limits.MaxShardsInPath = rowset.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(limits.MaxShardsInPath); + limits.MaxConsistentCopyTargets = rowset.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(limits.MaxConsistentCopyTargets); + limits.MaxPathElementLength = rowset.GetValueOrDefault<Schema::SubDomains::PathElementLength>(limits.MaxPathElementLength); + limits.ExtraPathSymbolsAllowed = rowset.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(limits.ExtraPathSymbolsAllowed); limits.MaxPQPartitions = rowset.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(limits.MaxPQPartitions); - domainInfo->SetSchemeLimits(limits); + domainInfo->SetSchemeLimits(limits); if (rowset.HaveValue<Schema::SubDomains::DeclaredSchemeQuotas>()) { NKikimrSubDomains::TSchemeQuotas declaredSchemeQuotas; @@ -1560,57 +1560,57 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (domainInfo->GetDiskQuotaExceeded()) { Self->ChangeDiskSpaceQuotaExceeded(+1); } - } - - if (!rowset.Next()) - return false; - } - - // Read SubDomainsAlterData - { - auto rowset = db.Table<Schema::SubDomainsAlterData>().Range().Select(); - if (!rowset.IsReady()) - return false; - while (!rowset.EndOfSet()) { - TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainsAlterData::PathId>(); - TPathId pathId(selfId, localPathId); - - ui64 alterVersion = rowset.GetValue<Schema::SubDomainsAlterData::AlterVersion>(); - ui64 planResolution = rowset.GetValue<Schema::SubDomainsAlterData::PlanResolution>(); - ui32 timeCastBukets = rowset.GetValue<Schema::SubDomainsAlterData::TimeCastBuckets>(); - - TPathId resourcesDomainId = TPathId( - rowset.GetValue<Schema::SubDomainsAlterData::ResourcesDomainOwnerPathId>(), - rowset.GetValue<Schema::SubDomainsAlterData::ResourcesDomainLocalPathId>()); - if (resourcesDomainId && Self->IsDomainSchemeShard) { - // we cannot check that on TSS - Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); - } - - Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); - TPathElement::TPtr path = Self->PathsById.at(pathId); - Y_VERIFY_S(path->IsDomainRoot(), "Path is not a subdomain, pathId: " << pathId); - - Y_VERIFY(Self->SubDomains.contains(pathId)); - auto subdomainInfo = Self->SubDomains[pathId]; - Y_VERIFY(!subdomainInfo->GetAlter()); - - TSubDomainInfo::TPtr alter; - alter = new TSubDomainInfo( - alterVersion, - planResolution, - timeCastBukets, - resourcesDomainId); - - TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomainsAlterData::SharedHiveId>(); - alter->SetSharedHive(sharedHiveId); - - alter->SetSchemeLimits(subdomainInfo->GetSchemeLimits()); // do not change SchemeLimits - - if (Self->IsDomainSchemeShard && path->IsRoot()) { - alter->InitializeAsGlobal(Self->CreateRootProcessingParams(ctx)); - } - + } + + if (!rowset.Next()) + return false; + } + + // Read SubDomainsAlterData + { + auto rowset = db.Table<Schema::SubDomainsAlterData>().Range().Select(); + if (!rowset.IsReady()) + return false; + while (!rowset.EndOfSet()) { + TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainsAlterData::PathId>(); + TPathId pathId(selfId, localPathId); + + ui64 alterVersion = rowset.GetValue<Schema::SubDomainsAlterData::AlterVersion>(); + ui64 planResolution = rowset.GetValue<Schema::SubDomainsAlterData::PlanResolution>(); + ui32 timeCastBukets = rowset.GetValue<Schema::SubDomainsAlterData::TimeCastBuckets>(); + + TPathId resourcesDomainId = TPathId( + rowset.GetValue<Schema::SubDomainsAlterData::ResourcesDomainOwnerPathId>(), + rowset.GetValue<Schema::SubDomainsAlterData::ResourcesDomainLocalPathId>()); + if (resourcesDomainId && Self->IsDomainSchemeShard) { + // we cannot check that on TSS + Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); + } + + Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); + TPathElement::TPtr path = Self->PathsById.at(pathId); + Y_VERIFY_S(path->IsDomainRoot(), "Path is not a subdomain, pathId: " << pathId); + + Y_VERIFY(Self->SubDomains.contains(pathId)); + auto subdomainInfo = Self->SubDomains[pathId]; + Y_VERIFY(!subdomainInfo->GetAlter()); + + TSubDomainInfo::TPtr alter; + alter = new TSubDomainInfo( + alterVersion, + planResolution, + timeCastBukets, + resourcesDomainId); + + TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomainsAlterData::SharedHiveId>(); + alter->SetSharedHive(sharedHiveId); + + alter->SetSchemeLimits(subdomainInfo->GetSchemeLimits()); // do not change SchemeLimits + + if (Self->IsDomainSchemeShard && path->IsRoot()) { + alter->InitializeAsGlobal(Self->CreateRootProcessingParams(ctx)); + } + if (rowset.HaveValue<Schema::SubDomainsAlterData::DeclaredSchemeQuotas>()) { NKikimrSubDomains::TSchemeQuotas declaredSchemeQuotas; Y_VERIFY(ParseFromStringNoSizeLimit(declaredSchemeQuotas, rowset.GetValue<Schema::SubDomainsAlterData::DeclaredSchemeQuotas>())); @@ -1623,90 +1623,90 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { alter->SetDatabaseQuotas(databaseQuotas); } - subdomainInfo->SetAlter(alter); - - if (!rowset.Next()) - return false; - } - } - - // Set ResourcesDomainId for older subdomains & check validity - for (auto [id, subDomain] : Self->SubDomains) { - auto alter = subDomain->GetAlter(); - - if (!subDomain->GetResourcesDomainId() && (!alter || !alter->GetResourcesDomainId())) { - if (Self->IsDomainSchemeShard) { - subDomain->SetResourcesDomainId(id); - } else { - subDomain->SetResourcesDomainId(Self->ParentDomainId); - } - - continue; - } - - if (!Self->IsDomainSchemeShard) { - // we cannot check validity on TSS - continue; - } - - if (auto resourcesDomainId = subDomain->GetResourcesDomainId()) { - Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); - } - - if (!alter) { - continue; - } - - if (auto resourcesDomainId = alter->GetResourcesDomainId()) { - Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); - } - } - } - - // Read SubDomainShards - { - auto rowset = db.Table<Schema::SubDomainShards>().Range().Select(); - if (!rowset.IsReady()) - return false; - while (!rowset.EndOfSet()) { - TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainShards::PathId>(); - TPathId pathId(selfId, localPathId); - - TLocalShardIdx localShardIdx = rowset.GetValue<Schema::SubDomainShards::ShardIdx>(); - TShardIdx shardIdx = Self->MakeLocalId(localShardIdx); - - Y_VERIFY(Self->SubDomains.contains(pathId)); - Self->SubDomains[pathId]->AddPrivateShard(shardIdx); - - if (!rowset.Next()) - return false; - } - - // Read SubDomainShardsAlterData - { - auto rowset = db.Table<Schema::SubDomainShardsAlterData>().Range().Select(); - if (!rowset.IsReady()) - return false; - while (!rowset.EndOfSet()) { - TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainShardsAlterData::PathId>(); - TPathId pathId(selfId, localPathId); - - TLocalShardIdx localShardIdx = rowset.GetValue<Schema::SubDomainShardsAlterData::ShardIdx>(); - TShardIdx shardIdx = Self->MakeLocalId(localShardIdx); - - Y_VERIFY(Self->SubDomains.contains(pathId)); - auto subdomainInfo = Self->SubDomains[pathId]; - Y_VERIFY(subdomainInfo->GetAlter()); - subdomainInfo->GetAlter()->AddPrivateShard(shardIdx); - - if (!rowset.Next()) - return false; - } - } - } - + subdomainInfo->SetAlter(alter); + + if (!rowset.Next()) + return false; + } + } + + // Set ResourcesDomainId for older subdomains & check validity + for (auto [id, subDomain] : Self->SubDomains) { + auto alter = subDomain->GetAlter(); + + if (!subDomain->GetResourcesDomainId() && (!alter || !alter->GetResourcesDomainId())) { + if (Self->IsDomainSchemeShard) { + subDomain->SetResourcesDomainId(id); + } else { + subDomain->SetResourcesDomainId(Self->ParentDomainId); + } + + continue; + } + + if (!Self->IsDomainSchemeShard) { + // we cannot check validity on TSS + continue; + } + + if (auto resourcesDomainId = subDomain->GetResourcesDomainId()) { + Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); + } + + if (!alter) { + continue; + } + + if (auto resourcesDomainId = alter->GetResourcesDomainId()) { + Y_VERIFY_S(Self->SubDomains.contains(resourcesDomainId), "Unknown ResourcesDomainId: " << resourcesDomainId); + } + } + } + + // Read SubDomainShards + { + auto rowset = db.Table<Schema::SubDomainShards>().Range().Select(); + if (!rowset.IsReady()) + return false; + while (!rowset.EndOfSet()) { + TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainShards::PathId>(); + TPathId pathId(selfId, localPathId); + + TLocalShardIdx localShardIdx = rowset.GetValue<Schema::SubDomainShards::ShardIdx>(); + TShardIdx shardIdx = Self->MakeLocalId(localShardIdx); + + Y_VERIFY(Self->SubDomains.contains(pathId)); + Self->SubDomains[pathId]->AddPrivateShard(shardIdx); + + if (!rowset.Next()) + return false; + } + + // Read SubDomainShardsAlterData + { + auto rowset = db.Table<Schema::SubDomainShardsAlterData>().Range().Select(); + if (!rowset.IsReady()) + return false; + while (!rowset.EndOfSet()) { + TLocalPathId localPathId = rowset.GetValue<Schema::SubDomainShardsAlterData::PathId>(); + TPathId pathId(selfId, localPathId); + + TLocalShardIdx localShardIdx = rowset.GetValue<Schema::SubDomainShardsAlterData::ShardIdx>(); + TShardIdx shardIdx = Self->MakeLocalId(localShardIdx); + + Y_VERIFY(Self->SubDomains.contains(pathId)); + auto subdomainInfo = Self->SubDomains[pathId]; + Y_VERIFY(subdomainInfo->GetAlter()); + subdomainInfo->GetAlter()->AddPrivateShard(shardIdx); + + if (!rowset.Next()) + return false; + } + } + } + // Read SubDomainSchemeQuotas - { + { auto rowset = db.Table<Schema::SubDomainSchemeQuotas>().Range().Select(); if (!rowset.IsReady()) return false; @@ -1790,20 +1790,20 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->PartitioningVersion = std::get<6>(rec); - TString ttlSettings = std::get<7>(rec); - if (ttlSettings) { - bool parseOk = ParseFromStringNoSizeLimit(tableInfo->MutableTTLSettings(), ttlSettings); - Y_VERIFY(parseOk); - } - - tableInfo->IsBackup = std::get<8>(rec); - + TString ttlSettings = std::get<7>(rec); + if (ttlSettings) { + bool parseOk = ParseFromStringNoSizeLimit(tableInfo->MutableTTLSettings(), ttlSettings); + Y_VERIFY(parseOk); + } + + tableInfo->IsBackup = std::get<8>(rec); + Self->Tables[pathId] = tableInfo; Self->IncrementPathDbRefCount(pathId); - if (tableInfo->IsTTLEnabled()) { - Self->TTLEnabledTables[pathId] = tableInfo; - Self->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); - } + if (tableInfo->IsTTLEnabled()) { + Self->TTLEnabledTables[pathId] = tableInfo; + Self->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); + } } } @@ -2008,14 +2008,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TPathId prevTableId; TVector<TTableShardInfo> partitions; - const auto now = ctx.Now(); + const auto now = ctx.Now(); for (auto& rec: tablePartitions) { TPathId tableId = std::get<0>(rec); ui64 id = std::get<1>(rec); TString rangeEnd = std::get<2>(rec); TShardIdx datashardIdx = std::get<3>(rec); - ui64 lastCondErase = std::get<4>(rec); - ui64 nextCondErase = std::get<5>(rec); + ui64 lastCondErase = std::get<4>(rec); + ui64 nextCondErase = std::get<5>(rec); if (tableId != prevTableId) { if (prevTableId) { @@ -2033,20 +2033,20 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (partitions.size() <= id) { partitions.resize(id+1); } - - partitions[id] = TTableShardInfo(datashardIdx, rangeEnd, lastCondErase, nextCondErase); - - if (Self->TTLEnabledTables.contains(tableId) && partitions[id].LastCondErase) { - auto& lag = partitions[id].LastCondEraseLag; - if (now >= partitions[id].LastCondErase) { - lag = now - partitions[id].LastCondErase; - } else { - lag = TDuration::Zero(); - } - - Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); - } - + + partitions[id] = TTableShardInfo(datashardIdx, rangeEnd, lastCondErase, nextCondErase); + + if (Self->TTLEnabledTables.contains(tableId) && partitions[id].LastCondErase) { + auto& lag = partitions[id].LastCondEraseLag; + if (now >= partitions[id].LastCondErase) { + lag = now - partitions[id].LastCondErase; + } else { + lag = TDuration::Zero(); + } + + Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); + } + // TODO: check rangeEnd validity // TODO: check datashard idx existence } @@ -2087,41 +2087,41 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - // Read partition stats - { - auto rowSet = db.Table<Schema::TablePartitionStats>().Range().Select(); - if (!rowSet.IsReady()) { - return false; - } - - TPathId prevTableId; - - while (!rowSet.EndOfSet()) { - const TPathId tableId = TPathId( - rowSet.GetValue<Schema::TablePartitionStats::TableOwnerId>(), - rowSet.GetValue<Schema::TablePartitionStats::TableLocalId>()); - - if (tableId != prevTableId) { - if (prevTableId) { - Y_VERIFY(Self->Tables.contains(prevTableId)); - TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); + // Read partition stats + { + auto rowSet = db.Table<Schema::TablePartitionStats>().Range().Select(); + if (!rowSet.IsReady()) { + return false; + } + + TPathId prevTableId; + + while (!rowSet.EndOfSet()) { + const TPathId tableId = TPathId( + rowSet.GetValue<Schema::TablePartitionStats::TableOwnerId>(), + rowSet.GetValue<Schema::TablePartitionStats::TableLocalId>()); + + if (tableId != prevTableId) { + if (prevTableId) { + Y_VERIFY(Self->Tables.contains(prevTableId)); + TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); if (!tableInfo->IsBackup && !tableInfo->IsShardsStatsDetached()) { Self->ResolveDomainInfo(prevTableId)->AggrDiskSpaceUsage(Self, tableInfo->GetStats().Aggregated); - } - } - - prevTableId = tableId; - } - - Y_VERIFY(Self->Tables.contains(tableId)); - TTableInfo::TPtr tableInfo = Self->Tables.at(tableId); - - const ui64 partitionId = rowSet.GetValue<Schema::TablePartitionStats::PartitionId>(); - Y_VERIFY(partitionId < tableInfo->GetPartitions().size()); - - const TShardIdx shardIdx = tableInfo->GetPartitions()[partitionId].ShardIdx; - Y_VERIFY(shardIdx != InvalidShardIdx); - + } + } + + prevTableId = tableId; + } + + Y_VERIFY(Self->Tables.contains(tableId)); + TTableInfo::TPtr tableInfo = Self->Tables.at(tableId); + + const ui64 partitionId = rowSet.GetValue<Schema::TablePartitionStats::PartitionId>(); + Y_VERIFY(partitionId < tableInfo->GetPartitions().size()); + + const TShardIdx shardIdx = tableInfo->GetPartitions()[partitionId].ShardIdx; + Y_VERIFY(shardIdx != InvalidShardIdx); + if (Self->ShardInfos.contains(shardIdx)) { const TShardInfo& shardInfo = Self->ShardInfos.at(shardIdx); if (shardInfo.PathId != tableId) { @@ -2129,58 +2129,58 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - TTableInfo::TPartitionStats stats; - - stats.SeqNo = TMessageSeqNo( - rowSet.GetValue<Schema::TablePartitionStats::SeqNoGeneration>(), - rowSet.GetValue<Schema::TablePartitionStats::SeqNoRound>()); - - stats.RowCount = rowSet.GetValue<Schema::TablePartitionStats::RowCount>(); - stats.DataSize = rowSet.GetValue<Schema::TablePartitionStats::DataSize>(); - stats.IndexSize = rowSet.GetValue<Schema::TablePartitionStats::IndexSize>(); - - stats.LastAccessTime = TInstant::FromValue(rowSet.GetValue<Schema::TablePartitionStats::LastAccessTime>()); - stats.LastUpdateTime = TInstant::FromValue(rowSet.GetValue<Schema::TablePartitionStats::LastUpdateTime>()); - - stats.ImmediateTxCompleted = rowSet.GetValue<Schema::TablePartitionStats::ImmediateTxCompleted>(); - stats.PlannedTxCompleted = rowSet.GetValue<Schema::TablePartitionStats::PlannedTxCompleted>(); - stats.TxRejectedByOverload = rowSet.GetValue<Schema::TablePartitionStats::TxRejectedByOverload>(); - stats.TxRejectedBySpace = rowSet.GetValue<Schema::TablePartitionStats::TxRejectedBySpace>(); - stats.TxCompleteLag = TDuration::FromValue(rowSet.GetValue<Schema::TablePartitionStats::TxCompleteLag>()); - stats.InFlightTxCount = rowSet.GetValue<Schema::TablePartitionStats::InFlightTxCount>(); - - stats.RowUpdates = rowSet.GetValue<Schema::TablePartitionStats::RowUpdates>(); - stats.RowDeletes = rowSet.GetValue<Schema::TablePartitionStats::RowDeletes>(); - stats.RowReads = rowSet.GetValue<Schema::TablePartitionStats::RowReads>(); - stats.RangeReads = rowSet.GetValue<Schema::TablePartitionStats::RangeReads>(); - stats.RangeReadRows = rowSet.GetValue<Schema::TablePartitionStats::RangeReadRows>(); - + TTableInfo::TPartitionStats stats; + + stats.SeqNo = TMessageSeqNo( + rowSet.GetValue<Schema::TablePartitionStats::SeqNoGeneration>(), + rowSet.GetValue<Schema::TablePartitionStats::SeqNoRound>()); + + stats.RowCount = rowSet.GetValue<Schema::TablePartitionStats::RowCount>(); + stats.DataSize = rowSet.GetValue<Schema::TablePartitionStats::DataSize>(); + stats.IndexSize = rowSet.GetValue<Schema::TablePartitionStats::IndexSize>(); + + stats.LastAccessTime = TInstant::FromValue(rowSet.GetValue<Schema::TablePartitionStats::LastAccessTime>()); + stats.LastUpdateTime = TInstant::FromValue(rowSet.GetValue<Schema::TablePartitionStats::LastUpdateTime>()); + + stats.ImmediateTxCompleted = rowSet.GetValue<Schema::TablePartitionStats::ImmediateTxCompleted>(); + stats.PlannedTxCompleted = rowSet.GetValue<Schema::TablePartitionStats::PlannedTxCompleted>(); + stats.TxRejectedByOverload = rowSet.GetValue<Schema::TablePartitionStats::TxRejectedByOverload>(); + stats.TxRejectedBySpace = rowSet.GetValue<Schema::TablePartitionStats::TxRejectedBySpace>(); + stats.TxCompleteLag = TDuration::FromValue(rowSet.GetValue<Schema::TablePartitionStats::TxCompleteLag>()); + stats.InFlightTxCount = rowSet.GetValue<Schema::TablePartitionStats::InFlightTxCount>(); + + stats.RowUpdates = rowSet.GetValue<Schema::TablePartitionStats::RowUpdates>(); + stats.RowDeletes = rowSet.GetValue<Schema::TablePartitionStats::RowDeletes>(); + stats.RowReads = rowSet.GetValue<Schema::TablePartitionStats::RowReads>(); + stats.RangeReads = rowSet.GetValue<Schema::TablePartitionStats::RangeReads>(); + stats.RangeReadRows = rowSet.GetValue<Schema::TablePartitionStats::RangeReadRows>(); + TInstant now = AppData(ctx)->TimeProvider->Now(); stats.SetCurrentRawCpuUsage(rowSet.GetValue<Schema::TablePartitionStats::CPU>(), now); - stats.Memory = rowSet.GetValue<Schema::TablePartitionStats::Memory>(); - stats.Network = rowSet.GetValue<Schema::TablePartitionStats::Network>(); - stats.Storage = rowSet.GetValue<Schema::TablePartitionStats::Storage>(); - stats.ReadThroughput = rowSet.GetValue<Schema::TablePartitionStats::ReadThroughput>(); - stats.WriteThroughput = rowSet.GetValue<Schema::TablePartitionStats::WriteThroughput>(); - stats.ReadIops = rowSet.GetValue<Schema::TablePartitionStats::ReadIops>(); - stats.WriteIops = rowSet.GetValue<Schema::TablePartitionStats::WriteIops>(); - - tableInfo->UpdateShardStats(shardIdx, stats); - - if (!rowSet.Next()) { - return false; - } - } - - if (prevTableId) { - Y_VERIFY(Self->Tables.contains(prevTableId)); - TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); + stats.Memory = rowSet.GetValue<Schema::TablePartitionStats::Memory>(); + stats.Network = rowSet.GetValue<Schema::TablePartitionStats::Network>(); + stats.Storage = rowSet.GetValue<Schema::TablePartitionStats::Storage>(); + stats.ReadThroughput = rowSet.GetValue<Schema::TablePartitionStats::ReadThroughput>(); + stats.WriteThroughput = rowSet.GetValue<Schema::TablePartitionStats::WriteThroughput>(); + stats.ReadIops = rowSet.GetValue<Schema::TablePartitionStats::ReadIops>(); + stats.WriteIops = rowSet.GetValue<Schema::TablePartitionStats::WriteIops>(); + + tableInfo->UpdateShardStats(shardIdx, stats); + + if (!rowSet.Next()) { + return false; + } + } + + if (prevTableId) { + Y_VERIFY(Self->Tables.contains(prevTableId)); + TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId); if (!tableInfo->IsBackup && !tableInfo->IsShardsStatsDetached()) { Self->ResolveDomainInfo(prevTableId)->AggrDiskSpaceUsage(Self, tableInfo->GetStats().Aggregated); - } - } - } - + } + } + } + // Read channels binding { TChannelBindingRows channelBindingRows; @@ -2233,9 +2233,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { pqGroup->NextPartitionId = rowset.GetValueOrDefault<Schema::PersQueueGroups::NextPartitionId>(0); pqGroup->TotalGroupCount = rowset.GetValueOrDefault<Schema::PersQueueGroups::TotalGroupCount>(0); - const bool ok = pqGroup->FillKeySchema(pqGroup->TabletConfig); - Y_VERIFY(ok); - + const bool ok = pqGroup->FillKeySchema(pqGroup->TabletConfig); + Y_VERIFY(ok); + Self->PersQueueGroups[pathId] = pqGroup; Self->IncrementPathDbRefCount(pathId); @@ -2268,22 +2268,22 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TShardIdx shardIdx = Self->MakeLocalId(localShardIdx); pqInfo.AlterVersion = rowset.GetValue<Schema::PersQueues::AlterVersion>(); - if (rowset.HaveValue<Schema::PersQueues::RangeBegin>()) { - if (!pqInfo.KeyRange) { - pqInfo.KeyRange = {}; - } - - pqInfo.KeyRange->FromBound = rowset.GetValue<Schema::PersQueues::RangeBegin>(); - } - - if (rowset.HaveValue<Schema::PersQueues::RangeEnd>()) { - if (!pqInfo.KeyRange) { - pqInfo.KeyRange = {}; - } - - pqInfo.KeyRange->ToBound = rowset.GetValue<Schema::PersQueues::RangeEnd>(); - } - + if (rowset.HaveValue<Schema::PersQueues::RangeBegin>()) { + if (!pqInfo.KeyRange) { + pqInfo.KeyRange = {}; + } + + pqInfo.KeyRange->FromBound = rowset.GetValue<Schema::PersQueues::RangeBegin>(); + } + + if (rowset.HaveValue<Schema::PersQueues::RangeEnd>()) { + if (!pqInfo.KeyRange) { + pqInfo.KeyRange = {}; + } + + pqInfo.KeyRange->ToBound = rowset.GetValue<Schema::PersQueues::RangeEnd>(); + } + auto it = Self->PersQueueGroups.find(pathId); Y_VERIFY(it != Self->PersQueueGroups.end()); Y_VERIFY(it->second); @@ -2321,11 +2321,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { alterData->AlterVersion = rowset.GetValue<Schema::PersQueueGroupAlters::AlterVersion>(); alterData->TotalGroupCount = rowset.GetValue<Schema::PersQueueGroupAlters::TotalGroupCount>(); alterData->NextPartitionId = rowset.GetValueOrDefault<Schema::PersQueueGroupAlters::NextPartitionId>(alterData->TotalGroupCount); - alterData->BootstrapConfig = rowset.GetValue<Schema::PersQueueGroupAlters::BootstrapConfig>(); + alterData->BootstrapConfig = rowset.GetValue<Schema::PersQueueGroupAlters::BootstrapConfig>(); + + const bool ok = alterData->FillKeySchema(alterData->TabletConfig); + Y_VERIFY(ok); - const bool ok = alterData->FillKeySchema(alterData->TabletConfig); - Y_VERIFY(ok); - auto it = Self->PersQueueGroups.find(pathId); Y_VERIFY(it != Self->PersQueueGroups.end()); @@ -2717,92 +2717,92 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - // Read CdcStream - { - auto rowset = db.Table<Schema::CdcStream>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - auto pathId = TPathId( - TOwnerId(rowset.GetValue<Schema::CdcStream::OwnerPathId>()), - TLocalPathId(rowset.GetValue<Schema::CdcStream::LocalPathId>()) - ); - auto alterVersion = rowset.GetValue<Schema::CdcStream::AlterVersion>(); - auto mode = rowset.GetValue<Schema::CdcStream::Mode>(); - auto state = rowset.GetValue<Schema::CdcStream::State>(); - - Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); - auto path = Self->PathsById.at(pathId); - - Y_VERIFY_S(path->IsCdcStream(), "Path is not a cdc stream" - << ", pathId: " << pathId + // Read CdcStream + { + auto rowset = db.Table<Schema::CdcStream>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + auto pathId = TPathId( + TOwnerId(rowset.GetValue<Schema::CdcStream::OwnerPathId>()), + TLocalPathId(rowset.GetValue<Schema::CdcStream::LocalPathId>()) + ); + auto alterVersion = rowset.GetValue<Schema::CdcStream::AlterVersion>(); + auto mode = rowset.GetValue<Schema::CdcStream::Mode>(); + auto state = rowset.GetValue<Schema::CdcStream::State>(); + + Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); + auto path = Self->PathsById.at(pathId); + + Y_VERIFY_S(path->IsCdcStream(), "Path is not a cdc stream" + << ", pathId: " << pathId << ", path type: " << NKikimrSchemeOp::EPathType_Name(path->PathType)); - - Y_VERIFY(!Self->CdcStreams.contains(pathId)); - Self->CdcStreams[pathId] = new TCdcStreamInfo(alterVersion, mode, state); - Self->IncrementPathDbRefCount(pathId); - - if (!rowset.Next()) { - return false; - } - } - } - - // Read CdcStreamAlterData - { - auto rowset = db.Table<Schema::CdcStreamAlterData>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - auto pathId = TPathId( - TOwnerId(rowset.GetValue<Schema::CdcStreamAlterData::OwnerPathId>()), - TLocalPathId(rowset.GetValue<Schema::CdcStreamAlterData::LocalPathId>()) - ); - - auto alterVersion = rowset.GetValue<Schema::CdcStreamAlterData::AlterVersion>(); - auto mode = rowset.GetValue<Schema::CdcStreamAlterData::Mode>(); - auto state = rowset.GetValue<Schema::CdcStreamAlterData::State>(); - - Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); - auto path = Self->PathsById.at(pathId); - - Y_VERIFY_S(path->IsCdcStream(), "Path is not a cdc stream" - << ", pathId: " << pathId + + Y_VERIFY(!Self->CdcStreams.contains(pathId)); + Self->CdcStreams[pathId] = new TCdcStreamInfo(alterVersion, mode, state); + Self->IncrementPathDbRefCount(pathId); + + if (!rowset.Next()) { + return false; + } + } + } + + // Read CdcStreamAlterData + { + auto rowset = db.Table<Schema::CdcStreamAlterData>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + auto pathId = TPathId( + TOwnerId(rowset.GetValue<Schema::CdcStreamAlterData::OwnerPathId>()), + TLocalPathId(rowset.GetValue<Schema::CdcStreamAlterData::LocalPathId>()) + ); + + auto alterVersion = rowset.GetValue<Schema::CdcStreamAlterData::AlterVersion>(); + auto mode = rowset.GetValue<Schema::CdcStreamAlterData::Mode>(); + auto state = rowset.GetValue<Schema::CdcStreamAlterData::State>(); + + Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); + auto path = Self->PathsById.at(pathId); + + Y_VERIFY_S(path->IsCdcStream(), "Path is not a cdc stream" + << ", pathId: " << pathId << ", path type: " << NKikimrSchemeOp::EPathType_Name(path->PathType)); - - if (!Self->CdcStreams.contains(pathId)) { - Y_VERIFY(alterVersion == 1); - Self->CdcStreams[pathId] = TCdcStreamInfo::New(mode); - Self->IncrementPathDbRefCount(pathId); - } - - auto stream = Self->CdcStreams.at(pathId); - Y_VERIFY(stream->AlterData == nullptr); - Y_VERIFY(stream->AlterVersion < alterVersion); - stream->AlterData = new TCdcStreamInfo(alterVersion, mode, state); - - Y_VERIFY_S(Self->PathsById.contains(path->ParentPathId), "Parent path is not found" - << ", cdc stream pathId: " << pathId - << ", parent pathId: " << path->ParentPathId); - auto parent = Self->PathsById.at(path->ParentPathId); - - Y_VERIFY_S(parent->IsTable(), "Parent path is not a table" - << ", cdc stream pathId: " << pathId - << ", parent pathId: " << path->ParentPathId); - Y_VERIFY_S(Self->Tables.contains(path->ParentPathId), "Parent path is not found in Tables map" - << ", cdc stream pathId: " << pathId - << ", parent pathId: " << path->ParentPathId); - - if (!rowset.Next()) { - return false; - } - } - } - + + if (!Self->CdcStreams.contains(pathId)) { + Y_VERIFY(alterVersion == 1); + Self->CdcStreams[pathId] = TCdcStreamInfo::New(mode); + Self->IncrementPathDbRefCount(pathId); + } + + auto stream = Self->CdcStreams.at(pathId); + Y_VERIFY(stream->AlterData == nullptr); + Y_VERIFY(stream->AlterVersion < alterVersion); + stream->AlterData = new TCdcStreamInfo(alterVersion, mode, state); + + Y_VERIFY_S(Self->PathsById.contains(path->ParentPathId), "Parent path is not found" + << ", cdc stream pathId: " << pathId + << ", parent pathId: " << path->ParentPathId); + auto parent = Self->PathsById.at(path->ParentPathId); + + Y_VERIFY_S(parent->IsTable(), "Parent path is not a table" + << ", cdc stream pathId: " << pathId + << ", parent pathId: " << path->ParentPathId); + Y_VERIFY_S(Self->Tables.contains(path->ParentPathId), "Parent path is not found in Tables map" + << ", cdc stream pathId: " << pathId + << ", parent pathId: " << path->ParentPathId); + + if (!rowset.Next()) { + return false; + } + } + } + // Read DomainsPools { auto rowset = db.Table<Schema::StoragePools>().Range().Select(); @@ -3125,8 +3125,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TString extraData = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::ExtraBytes>(""); txState.StartTime = TInstant::MicroSeconds(txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::StartTime>()); - txState.DataTotalSize = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::DataTotalSize>(0); - txState.Cancel = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::CancelBackup>(false); + txState.DataTotalSize = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::DataTotalSize>(0); + txState.Cancel = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::CancelBackup>(false); txState.BuildIndexId = txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::BuildIndexId>(); txState.SourcePathId = TPathId(txInFlightRowset.GetValueOrDefault<Schema::TxInFlightV2::SourceOwnerId>(), @@ -3157,9 +3157,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (txState.TxType == TTxState::TxCreateSubDomain) { Y_VERIFY(Self->SubDomains.contains(txState.TargetPathId)); auto subDomainInfo = Self->SubDomains.at(txState.TargetPathId); - if (txState.State <= TTxState::Propose) { - Y_VERIFY(subDomainInfo->GetAlter()); - } + if (txState.State <= TTxState::Propose) { + Y_VERIFY(subDomainInfo->GetAlter()); + } } else if (txState.TxType == TTxState::TxSplitTablePartition || txState.TxType == TTxState::TxMergeTablePartition) { Y_VERIFY(!extraData.empty(), "Split Tx must have non-empty split description"); txState.SplitDescription = std::make_shared<NKikimrTxDataShard::TSplitMergeDescription>(); @@ -3187,24 +3187,24 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->InitAlterData(); tableInfo->DeserializeAlterExtraData(extraData); } - } else if (txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore) { - auto byShardBackupStatus = db.Table<Schema::ShardBackupStatus>().Range(operationId.GetTxId()).Select(); - auto byMigratedShardBackupStatus = db.Table<Schema::MigratedShardBackupStatus>().Range(operationId.GetTxId()).Select(); - auto byTxShardStatus = db.Table<Schema::TxShardStatus>().Range(operationId.GetTxId()).Select(); + } else if (txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore) { + auto byShardBackupStatus = db.Table<Schema::ShardBackupStatus>().Range(operationId.GetTxId()).Select(); + auto byMigratedShardBackupStatus = db.Table<Schema::MigratedShardBackupStatus>().Range(operationId.GetTxId()).Select(); + auto byTxShardStatus = db.Table<Schema::TxShardStatus>().Range(operationId.GetTxId()).Select(); - TShardBackupStatusRows statuses; - if (!LoadBackupStatusesImpl(statuses, byShardBackupStatus, byMigratedShardBackupStatus, byTxShardStatus)) { + TShardBackupStatusRows statuses; + if (!LoadBackupStatusesImpl(statuses, byShardBackupStatus, byMigratedShardBackupStatus, byTxShardStatus)) { return false; } - for (auto& rec : statuses) { - auto shardIdx = std::get<1>(rec); - auto success = std::get<2>(rec); - auto error = std::get<3>(rec); - auto bytes = std::get<4>(rec); - auto rows = std::get<5>(rec); + for (auto& rec : statuses) { + auto shardIdx = std::get<1>(rec); + auto success = std::get<2>(rec); + auto error = std::get<3>(rec); + auto bytes = std::get<4>(rec); + auto rows = std::get<5>(rec); - txState.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); + txState.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); } } else if (txState.TxType == TTxState::TxForceDropSubDomain || txState.TxType == TTxState::TxForceDropExtSubDomain) { forceDropOpIds.push_back(operationId); @@ -3320,10 +3320,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { txState->TxType != TTxState::TxDropOlapTable && txState->TxType != TTxState::TxCreateSequence && txState->TxType != TTxState::TxAlterSequence && - txState->TxType != TTxState::TxDropSequence && - txState->TxType != TTxState::TxCreateReplication && - txState->TxType != TTxState::TxAlterReplication && - txState->TxType != TTxState::TxDropReplication) + txState->TxType != TTxState::TxDropSequence && + txState->TxType != TTxState::TxCreateReplication && + txState->TxType != TTxState::TxAlterReplication && + txState->TxType != TTxState::TxDropReplication) { Y_VERIFY_S(txState->TxType == TTxState::TxCopyTable, "Only CopyTable Tx can have participating shards from a different table" << ", txId: " << operationId.GetTxId() @@ -3435,77 +3435,77 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TPathId pathId = std::get<0>(rec); TString tableName = std::get<1>(rec); TString ytSerializedSettings = std::get<2>(rec); - TString s3SerializedSettings = std::get<3>(rec); - TString scanSettings = std::get<4>(rec); - bool needToBill = std::get<5>(rec); - TString tableDesc = std::get<6>(rec); - ui32 nRetries = std::get<7>(rec); + TString s3SerializedSettings = std::get<3>(rec); + TString scanSettings = std::get<4>(rec); + bool needToBill = std::get<5>(rec); + TString tableDesc = std::get<6>(rec); + ui32 nRetries = std::get<7>(rec); Y_VERIFY(tableName.size() > 0); TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); Y_VERIFY(tableInfo.Get() != nullptr); - + tableInfo->BackupSettings.SetTableName(tableName); - tableInfo->BackupSettings.SetNeedToBill(needToBill); - tableInfo->BackupSettings.SetNumberOfRetries(nRetries); - - if (ytSerializedSettings) { - auto settings = tableInfo->BackupSettings.MutableYTSettings(); - Y_VERIFY(ParseFromStringNoSizeLimit(*settings, ytSerializedSettings)); - } else if (s3SerializedSettings) { - auto settings = tableInfo->BackupSettings.MutableS3Settings(); - Y_VERIFY(ParseFromStringNoSizeLimit(*settings, s3SerializedSettings)); - } else { - Y_FAIL("Unknown settings"); - } - - if (scanSettings) { - auto settings = tableInfo->BackupSettings.MutableScanSettings(); - Y_VERIFY(ParseFromStringNoSizeLimit(*settings, scanSettings)); - } - - if (tableDesc) { - auto desc = tableInfo->BackupSettings.MutableTable(); - Y_VERIFY(ParseFromStringNoSizeLimit(*desc, tableDesc)); - } - + tableInfo->BackupSettings.SetNeedToBill(needToBill); + tableInfo->BackupSettings.SetNumberOfRetries(nRetries); + + if (ytSerializedSettings) { + auto settings = tableInfo->BackupSettings.MutableYTSettings(); + Y_VERIFY(ParseFromStringNoSizeLimit(*settings, ytSerializedSettings)); + } else if (s3SerializedSettings) { + auto settings = tableInfo->BackupSettings.MutableS3Settings(); + Y_VERIFY(ParseFromStringNoSizeLimit(*settings, s3SerializedSettings)); + } else { + Y_FAIL("Unknown settings"); + } + + if (scanSettings) { + auto settings = tableInfo->BackupSettings.MutableScanSettings(); + Y_VERIFY(ParseFromStringNoSizeLimit(*settings, scanSettings)); + } + + if (tableDesc) { + auto desc = tableInfo->BackupSettings.MutableTable(); + Y_VERIFY(ParseFromStringNoSizeLimit(*desc, tableDesc)); + } + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Loaded backup settings" << ", pathId: " << pathId << ", tablename: " << tableName.data()); } } - // Read restore tasks + // Read restore tasks { - auto rowSet = db.Table<Schema::RestoreTasks>().Range().Select(); - if (!rowSet.IsReady()) { - return false; - } - - while (!rowSet.EndOfSet()) { - auto pathId = TPathId( - rowSet.GetValue<Schema::RestoreTasks::OwnerPathId>(), - rowSet.GetValue<Schema::RestoreTasks::LocalPathId>()); - auto task = rowSet.GetValue<Schema::RestoreTasks::Task>(); - - TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); - Y_VERIFY(tableInfo.Get() != nullptr); - Y_VERIFY(ParseFromStringNoSizeLimit(tableInfo->RestoreSettings, task)); - - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Loaded restore task" - << ", pathId: " << pathId); - - if (!rowSet.Next()) { - return false; - } - } - } - + auto rowSet = db.Table<Schema::RestoreTasks>().Range().Select(); + if (!rowSet.IsReady()) { + return false; + } + + while (!rowSet.EndOfSet()) { + auto pathId = TPathId( + rowSet.GetValue<Schema::RestoreTasks::OwnerPathId>(), + rowSet.GetValue<Schema::RestoreTasks::LocalPathId>()); + auto task = rowSet.GetValue<Schema::RestoreTasks::Task>(); + + TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); + Y_VERIFY(tableInfo.Get() != nullptr); + Y_VERIFY(ParseFromStringNoSizeLimit(tableInfo->RestoreSettings, task)); + + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Loaded restore task" + << ", pathId: " << pathId); + + if (!rowSet.Next()) { + return false; + } + } + } + // Read security state NLoginProto::TSecurityState securityState; - { + { auto rowset = db.Table<Schema::LoginKeys>().Select(); if (!rowset.IsReady()) { @@ -3579,17 +3579,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { statusesByTxId[txId].push_back(rec); } - TCompletedBackupRestoreRows history; - if (!LoadBackupRestoreHistory(db, history)) { + TCompletedBackupRestoreRows history; + if (!LoadBackupRestoreHistory(db, history)) { return false; } LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for CompletedBackup" - << ", readed records: " << history.size() + << ", readed records: " << history.size() << ", at schemeshard: " << Self->TabletID()); - for (auto& rec: history) { + for (auto& rec: history) { auto pathId = std::get<0>(rec); auto txId = std::get<1>(rec); auto completeTime = std::get<2>(rec); @@ -3597,15 +3597,15 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto successShardsCount = std::get<3>(rec); auto totalShardCount = std::get<4>(rec); auto startTime = std::get<5>(rec); - auto dataSize = std::get<6>(rec); - auto kind = static_cast<TTableInfo::TBackupRestoreResult::EKind>(std::get<7>(rec)); + auto dataSize = std::get<6>(rec); + auto kind = static_cast<TTableInfo::TBackupRestoreResult::EKind>(std::get<7>(rec)); - TTableInfo::TBackupRestoreResult info; - info.CompletionDateTime = completeTime; - info.TotalShardCount = totalShardCount; - info.SuccessShardCount = successShardsCount; - info.StartDateTime = startTime; - info.DataTotalSize = dataSize; + TTableInfo::TBackupRestoreResult info; + info.CompletionDateTime = completeTime; + info.TotalShardCount = totalShardCount; + info.SuccessShardCount = successShardsCount; + info.StartDateTime = startTime; + info.DataTotalSize = dataSize; if (!Self->Tables.FindPtr(pathId)) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -3614,29 +3614,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { << ", txid: " << txId); continue; } - + TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); if (statusesByTxId.contains(txId)) { for (auto& recByTxId: statusesByTxId.at(txId)) { auto shardIdx = std::get<1>(recByTxId); - auto success = std::get<2>(recByTxId); - auto error = std::get<3>(recByTxId); - auto bytes = std::get<4>(recByTxId); - auto rows = std::get<5>(recByTxId); + auto success = std::get<2>(recByTxId); + auto error = std::get<3>(recByTxId); + auto bytes = std::get<4>(recByTxId); + auto rows = std::get<5>(recByTxId); - info.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); + info.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); } } - switch (kind) { - case TTableInfo::TBackupRestoreResult::EKind::Backup: - tableInfo->BackupHistory[txId] = std::move(info); - break; - case TTableInfo::TBackupRestoreResult::EKind::Restore: - tableInfo->RestoreHistory[txId] = std::move(info); - break; - } + switch (kind) { + case TTableInfo::TBackupRestoreResult::EKind::Backup: + tableInfo->BackupHistory[txId] = std::move(info); + break; + case TTableInfo::TBackupRestoreResult::EKind::Restore: + tableInfo->RestoreHistory[txId] = std::move(info); + break; + } LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Loaded completed backup status" @@ -3720,10 +3720,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Self->TabletCounters->Simple()[COUNTER_SEQUENCESHARD_COUNT].Add(1); domainInfo->AddSequenceShard(shardIdx); break; - case ETabletType::ReplicationController: - Self->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Add(1); - domainInfo->AddReplicationController(shardIdx); - break; + case ETabletType::ReplicationController: + Self->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Add(1); + domainInfo->AddReplicationController(shardIdx); + break; default: Y_FAIL_S("dont know how to interpret tablet type" << ", type id: " << (ui32)si.second.TabletType @@ -3803,8 +3803,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Self->TabletCounters->Simple()[COUNTER_OLAP_TABLE_COUNT].Add(1); } else if (path->IsSequence()) { Self->TabletCounters->Simple()[COUNTER_SEQUENCE_COUNT].Add(1); - } else if (path->IsReplication()) { - Self->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1); + } else if (path->IsReplication()) { + Self->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1); } path->ApplySpecialAttributes(); @@ -3828,235 +3828,235 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Self->TabletCounters->Simple()[inFlightCounter].Add(1); } - // Publications - { + // Publications + { TPublicationsRows publicationRows; if (!LoadPublications(db, publicationRows)) { - return false; - } - + return false; + } + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for Publications" << ", readed records: " << publicationRows.size() << ", at schemeshard: " << Self->TabletID()); - + for (auto& rec: publicationRows) { TTxId txId = std::get<0>(rec); TPathId pathId = std::get<1>(rec); ui64 version = std::get<2>(rec); - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Resume publishing for paths" - << ", tx: " << txId - << ", path id: " << pathId - << ", version: " << version - << ", at schemeshard: " << Self->TabletID()); - - if (Self->Operations.contains(txId)) { - TOperation::TPtr operation = Self->Operations.at(txId); - operation->AddPublishingPath(pathId, version); - } else { - Self->Publications[txId].Paths.emplace(pathId, version); - } - - Publications[txId].push_back(pathId); + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Resume publishing for paths" + << ", tx: " << txId + << ", path id: " << pathId + << ", version: " << version + << ", at schemeshard: " << Self->TabletID()); + + if (Self->Operations.contains(txId)) { + TOperation::TPtr operation = Self->Operations.at(txId); + operation->AddPublishingPath(pathId, version); + } else { + Self->Publications[txId].Paths.emplace(pathId, version); + } + + Publications[txId].push_back(pathId); Self->IncrementPathDbRefCount(pathId); - } - } - - // Read exports - { - // read main info - { - auto rowset = db.Table<Schema::Exports>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 id = rowset.GetValue<Schema::Exports::Id>(); - TString uid = rowset.GetValue<Schema::Exports::Uid>(); - TExportInfo::EKind kind = static_cast<TExportInfo::EKind>(rowset.GetValueOrDefault<Schema::Exports::Kind>(0)); - TString settings = rowset.GetValue<Schema::Exports::Settings>(); + } + } + + // Read exports + { + // read main info + { + auto rowset = db.Table<Schema::Exports>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 id = rowset.GetValue<Schema::Exports::Id>(); + TString uid = rowset.GetValue<Schema::Exports::Uid>(); + TExportInfo::EKind kind = static_cast<TExportInfo::EKind>(rowset.GetValueOrDefault<Schema::Exports::Kind>(0)); + TString settings = rowset.GetValue<Schema::Exports::Settings>(); auto domainPathId = TPathId(rowset.GetValueOrDefault<Schema::Exports::DomainPathOwnerId>(selfId), rowset.GetValue<Schema::Exports::DomainPathId>()); - - TExportInfo::TPtr exportInfo = new TExportInfo(id, uid, kind, settings, domainPathId); - - if (rowset.HaveValue<Schema::Exports::UserSID>()) { - exportInfo->UserSID = rowset.GetValue<Schema::Exports::UserSID>(); - } - - ui32 items = rowset.GetValue<Schema::Exports::Items>(); - exportInfo->Items.resize(items); - + + TExportInfo::TPtr exportInfo = new TExportInfo(id, uid, kind, settings, domainPathId); + + if (rowset.HaveValue<Schema::Exports::UserSID>()) { + exportInfo->UserSID = rowset.GetValue<Schema::Exports::UserSID>(); + } + + ui32 items = rowset.GetValue<Schema::Exports::Items>(); + exportInfo->Items.resize(items); + exportInfo->ExportPathId = TPathId(rowset.GetValueOrDefault<Schema::Exports::ExportOwnerPathId>(selfId), rowset.GetValueOrDefault<Schema::Exports::ExportPathId>(InvalidLocalPathId)); if (exportInfo->ExportPathId.LocalPathId == InvalidLocalPathId) { exportInfo->ExportPathId = InvalidPathId; } - exportInfo->State = static_cast<TExportInfo::EState>(rowset.GetValue<Schema::Exports::State>()); - exportInfo->WaitTxId = rowset.GetValueOrDefault<Schema::Exports::WaitTxId>(InvalidTxId); - exportInfo->Issue = rowset.GetValueOrDefault<Schema::Exports::Issue>(TString()); - - Self->Exports[id] = exportInfo; - if (uid) { - Self->ExportsByUid[uid] = exportInfo; - } - - if (exportInfo->WaitTxId != InvalidTxId) { - Self->TxIdToExport[exportInfo->WaitTxId] = {id, Max<ui32>()}; - } - - if (exportInfo->IsInProgress()) { - ExportsToResume.push_back(exportInfo->Id); - } - - if (!rowset.Next()) { - return false; - } - } - } - - // read items info - { - auto rowset = db.Table<Schema::ExportItems>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 exportId = rowset.GetValue<Schema::ExportItems::ExportId>(); - Y_VERIFY_S(Self->Exports.contains(exportId), "Export not found" - << ": exportId# " << exportId); - - TExportInfo::TPtr exportInfo = Self->Exports.at(exportId); - - ui32 itemIdx = rowset.GetValue<Schema::ExportItems::Index>(); - Y_VERIFY_S(itemIdx < exportInfo->Items.size(), "Invalid item's index" - << ": exportId# " << exportId - << ", itemIdx# " << itemIdx); - - TExportInfo::TItem& item = exportInfo->Items[itemIdx]; - item.SourcePathName = rowset.GetValue<Schema::ExportItems::SourcePathName>(); - + exportInfo->State = static_cast<TExportInfo::EState>(rowset.GetValue<Schema::Exports::State>()); + exportInfo->WaitTxId = rowset.GetValueOrDefault<Schema::Exports::WaitTxId>(InvalidTxId); + exportInfo->Issue = rowset.GetValueOrDefault<Schema::Exports::Issue>(TString()); + + Self->Exports[id] = exportInfo; + if (uid) { + Self->ExportsByUid[uid] = exportInfo; + } + + if (exportInfo->WaitTxId != InvalidTxId) { + Self->TxIdToExport[exportInfo->WaitTxId] = {id, Max<ui32>()}; + } + + if (exportInfo->IsInProgress()) { + ExportsToResume.push_back(exportInfo->Id); + } + + if (!rowset.Next()) { + return false; + } + } + } + + // read items info + { + auto rowset = db.Table<Schema::ExportItems>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 exportId = rowset.GetValue<Schema::ExportItems::ExportId>(); + Y_VERIFY_S(Self->Exports.contains(exportId), "Export not found" + << ": exportId# " << exportId); + + TExportInfo::TPtr exportInfo = Self->Exports.at(exportId); + + ui32 itemIdx = rowset.GetValue<Schema::ExportItems::Index>(); + Y_VERIFY_S(itemIdx < exportInfo->Items.size(), "Invalid item's index" + << ": exportId# " << exportId + << ", itemIdx# " << itemIdx); + + TExportInfo::TItem& item = exportInfo->Items[itemIdx]; + item.SourcePathName = rowset.GetValue<Schema::ExportItems::SourcePathName>(); + item.SourcePathId.OwnerId = rowset.GetValueOrDefault<Schema::ExportItems::SourceOwnerPathId>(selfId); item.SourcePathId.LocalPathId = rowset.GetValue<Schema::ExportItems::SourcePathId>(); - item.State = static_cast<TExportInfo::EState>(rowset.GetValue<Schema::ExportItems::State>()); - item.WaitTxId = rowset.GetValueOrDefault<Schema::ExportItems::BackupTxId>(InvalidTxId); - item.Issue = rowset.GetValueOrDefault<Schema::ExportItems::Issue>(TString()); - - if (item.State <= TExportInfo::EState::Transferring && item.WaitTxId == InvalidTxId) { - exportInfo->PendingItems.push_back(itemIdx); - } else if (item.WaitTxId != InvalidTxId) { - Self->TxIdToExport[item.WaitTxId] = {exportId, itemIdx}; - } - - if (!rowset.Next()) { - return false; - } - } - } - } - - // Read imports - { - // read main info - { - auto rowset = db.Table<Schema::Imports>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 id = rowset.GetValue<Schema::Imports::Id>(); - TString uid = rowset.GetValue<Schema::Imports::Uid>(); - TImportInfo::EKind kind = static_cast<TImportInfo::EKind>(rowset.GetValue<Schema::Imports::Kind>()); - auto domainPathId = TPathId(rowset.GetValue<Schema::Imports::DomainPathOwnerId>(), - rowset.GetValue<Schema::Imports::DomainPathLocalId>()); - - Ydb::Import::ImportFromS3Settings settings; - Y_VERIFY(ParseFromStringNoSizeLimit(settings, rowset.GetValue<Schema::Imports::Settings>())); - - TImportInfo::TPtr importInfo = new TImportInfo(id, uid, kind, settings, domainPathId); - - if (rowset.HaveValue<Schema::Imports::UserSID>()) { - importInfo->UserSID = rowset.GetValue<Schema::Imports::UserSID>(); - } - - ui32 items = rowset.GetValue<Schema::Imports::Items>(); - importInfo->Items.resize(items); - - importInfo->State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::Imports::State>()); - importInfo->Issue = rowset.GetValueOrDefault<Schema::Imports::Issue>(TString()); - - Self->Imports[id] = importInfo; - if (uid) { - Self->ImportsByUid[uid] = importInfo; - } - - switch (importInfo->State) { - case TImportInfo::EState::Waiting: - case TImportInfo::EState::Cancellation: - ImportsToResume.push_back(importInfo->Id); - break; - default: - break; - } - - if (!rowset.Next()) { - return false; - } - } - } - - // read items info - { - auto rowset = db.Table<Schema::ImportItems>().Range().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - ui64 importId = rowset.GetValue<Schema::ImportItems::ImportId>(); - Y_VERIFY_S(Self->Imports.contains(importId), "Import not found" - << ": importId# " << importId); - - TImportInfo::TPtr importInfo = Self->Imports.at(importId); - - ui32 itemIdx = rowset.GetValue<Schema::ImportItems::Index>(); - Y_VERIFY_S(itemIdx < importInfo->Items.size(), "Invalid item's index" - << ": importId# " << importId - << ", itemIdx# " << itemIdx); - - TImportInfo::TItem& item = importInfo->Items[itemIdx]; - item.DstPathName = rowset.GetValue<Schema::ImportItems::DstPathName>(); - item.DstPathId = TPathId(rowset.GetValueOrDefault<Schema::ImportItems::DstPathOwnerId>(InvalidOwnerId), - rowset.GetValueOrDefault<Schema::ImportItems::DstPathLocalId>(InvalidLocalPathId)); - - if (rowset.HaveValue<Schema::ImportItems::Scheme>()) { - Ydb::Table::CreateTableRequest scheme; - Y_VERIFY(ParseFromStringNoSizeLimit(scheme, rowset.GetValue<Schema::ImportItems::Scheme>())); - item.Scheme = scheme; - } - - item.State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::ImportItems::State>()); - item.WaitTxId = rowset.GetValueOrDefault<Schema::ImportItems::WaitTxId>(InvalidTxId); - item.NextIndexIdx = rowset.GetValueOrDefault<Schema::ImportItems::NextIndexIdx>(0); - item.Issue = rowset.GetValueOrDefault<Schema::ImportItems::Issue>(TString()); - - if (item.WaitTxId != InvalidTxId) { - Self->TxIdToImport[item.WaitTxId] = {importId, itemIdx}; - } - - if (!rowset.Next()) { - return false; - } - } - } - } - + item.State = static_cast<TExportInfo::EState>(rowset.GetValue<Schema::ExportItems::State>()); + item.WaitTxId = rowset.GetValueOrDefault<Schema::ExportItems::BackupTxId>(InvalidTxId); + item.Issue = rowset.GetValueOrDefault<Schema::ExportItems::Issue>(TString()); + + if (item.State <= TExportInfo::EState::Transferring && item.WaitTxId == InvalidTxId) { + exportInfo->PendingItems.push_back(itemIdx); + } else if (item.WaitTxId != InvalidTxId) { + Self->TxIdToExport[item.WaitTxId] = {exportId, itemIdx}; + } + + if (!rowset.Next()) { + return false; + } + } + } + } + + // Read imports + { + // read main info + { + auto rowset = db.Table<Schema::Imports>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 id = rowset.GetValue<Schema::Imports::Id>(); + TString uid = rowset.GetValue<Schema::Imports::Uid>(); + TImportInfo::EKind kind = static_cast<TImportInfo::EKind>(rowset.GetValue<Schema::Imports::Kind>()); + auto domainPathId = TPathId(rowset.GetValue<Schema::Imports::DomainPathOwnerId>(), + rowset.GetValue<Schema::Imports::DomainPathLocalId>()); + + Ydb::Import::ImportFromS3Settings settings; + Y_VERIFY(ParseFromStringNoSizeLimit(settings, rowset.GetValue<Schema::Imports::Settings>())); + + TImportInfo::TPtr importInfo = new TImportInfo(id, uid, kind, settings, domainPathId); + + if (rowset.HaveValue<Schema::Imports::UserSID>()) { + importInfo->UserSID = rowset.GetValue<Schema::Imports::UserSID>(); + } + + ui32 items = rowset.GetValue<Schema::Imports::Items>(); + importInfo->Items.resize(items); + + importInfo->State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::Imports::State>()); + importInfo->Issue = rowset.GetValueOrDefault<Schema::Imports::Issue>(TString()); + + Self->Imports[id] = importInfo; + if (uid) { + Self->ImportsByUid[uid] = importInfo; + } + + switch (importInfo->State) { + case TImportInfo::EState::Waiting: + case TImportInfo::EState::Cancellation: + ImportsToResume.push_back(importInfo->Id); + break; + default: + break; + } + + if (!rowset.Next()) { + return false; + } + } + } + + // read items info + { + auto rowset = db.Table<Schema::ImportItems>().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + ui64 importId = rowset.GetValue<Schema::ImportItems::ImportId>(); + Y_VERIFY_S(Self->Imports.contains(importId), "Import not found" + << ": importId# " << importId); + + TImportInfo::TPtr importInfo = Self->Imports.at(importId); + + ui32 itemIdx = rowset.GetValue<Schema::ImportItems::Index>(); + Y_VERIFY_S(itemIdx < importInfo->Items.size(), "Invalid item's index" + << ": importId# " << importId + << ", itemIdx# " << itemIdx); + + TImportInfo::TItem& item = importInfo->Items[itemIdx]; + item.DstPathName = rowset.GetValue<Schema::ImportItems::DstPathName>(); + item.DstPathId = TPathId(rowset.GetValueOrDefault<Schema::ImportItems::DstPathOwnerId>(InvalidOwnerId), + rowset.GetValueOrDefault<Schema::ImportItems::DstPathLocalId>(InvalidLocalPathId)); + + if (rowset.HaveValue<Schema::ImportItems::Scheme>()) { + Ydb::Table::CreateTableRequest scheme; + Y_VERIFY(ParseFromStringNoSizeLimit(scheme, rowset.GetValue<Schema::ImportItems::Scheme>())); + item.Scheme = scheme; + } + + item.State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::ImportItems::State>()); + item.WaitTxId = rowset.GetValueOrDefault<Schema::ImportItems::WaitTxId>(InvalidTxId); + item.NextIndexIdx = rowset.GetValueOrDefault<Schema::ImportItems::NextIndexIdx>(0); + item.Issue = rowset.GetValueOrDefault<Schema::ImportItems::Issue>(TString()); + + if (item.WaitTxId != InvalidTxId) { + Self->TxIdToImport[item.WaitTxId] = {importId, itemIdx}; + } + + if (!rowset.Next()) { + return false; + } + } + } + } + // Read index build { // read main info @@ -4468,53 +4468,53 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - // Read replications - { - auto rowset = db.Table<Schema::Replications>().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - TPathId pathId = Self->MakeLocalId(rowset.GetValue<Schema::Replications::PathId>()); - ui64 alterVersion = rowset.GetValue<Schema::Replications::AlterVersion>(); - NKikimrSchemeOp::TReplicationDescription description; - Y_VERIFY(description.ParseFromString(rowset.GetValue<Schema::Replications::Description>())); - - TReplicationInfo::TPtr replicationInfo = new TReplicationInfo(alterVersion, std::move(description)); - Self->Replications[pathId] = replicationInfo; - Self->IncrementPathDbRefCount(pathId); - - if (!rowset.Next()) { - return false; - } - } - } - - // Read replication alters - { - auto rowset = db.Table<Schema::ReplicationsAlterData>().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - TPathId pathId = Self->MakeLocalId(rowset.GetValue<Schema::ReplicationsAlterData::PathId>()); - ui64 alterVersion = rowset.GetValue<Schema::ReplicationsAlterData::AlterVersion>(); - NKikimrSchemeOp::TReplicationDescription description; - Y_VERIFY(description.ParseFromString(rowset.GetValue<Schema::ReplicationsAlterData::Description>())); - - TReplicationInfo::TPtr alterData = new TReplicationInfo(alterVersion, std::move(description)); - Y_VERIFY_S(Self->Replications.contains(pathId), - "Cannot load alter for replication " << pathId); - Self->Replications[pathId]->AlterData = alterData; - - if (!rowset.Next()) { - return false; - } - } - } - + // Read replications + { + auto rowset = db.Table<Schema::Replications>().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + TPathId pathId = Self->MakeLocalId(rowset.GetValue<Schema::Replications::PathId>()); + ui64 alterVersion = rowset.GetValue<Schema::Replications::AlterVersion>(); + NKikimrSchemeOp::TReplicationDescription description; + Y_VERIFY(description.ParseFromString(rowset.GetValue<Schema::Replications::Description>())); + + TReplicationInfo::TPtr replicationInfo = new TReplicationInfo(alterVersion, std::move(description)); + Self->Replications[pathId] = replicationInfo; + Self->IncrementPathDbRefCount(pathId); + + if (!rowset.Next()) { + return false; + } + } + } + + // Read replication alters + { + auto rowset = db.Table<Schema::ReplicationsAlterData>().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + TPathId pathId = Self->MakeLocalId(rowset.GetValue<Schema::ReplicationsAlterData::PathId>()); + ui64 alterVersion = rowset.GetValue<Schema::ReplicationsAlterData::AlterVersion>(); + NKikimrSchemeOp::TReplicationDescription description; + Y_VERIFY(description.ParseFromString(rowset.GetValue<Schema::ReplicationsAlterData::Description>())); + + TReplicationInfo::TPtr alterData = new TReplicationInfo(alterVersion, std::move(description)); + Y_VERIFY_S(Self->Replications.contains(pathId), + "Cannot load alter for replication " << pathId); + Self->Replications[pathId]->AlterData = alterData; + + if (!rowset.Next()) { + return false; + } + } + } + for (auto& item : Self->Operations) { auto& operation = item.second; LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -4564,10 +4564,10 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { void Complete(const TActorContext &ctx) override { auto delayPublications = OnComplete.ExtractPublicationsToSchemeBoard(); //there no Populator exist jet - for (auto& [txId, pathIds] : Publications) { - std::move(pathIds.begin(), pathIds.end(), std::back_inserter(delayPublications[txId])); - } - + for (auto& [txId, pathIds] : Publications) { + std::move(pathIds.begin(), pathIds.end(), std::back_inserter(delayPublications[txId])); + } + OnComplete.ApplyOnComplete(Self, ctx); if (!Self->IsShemeShardConfigured()) { diff --git a/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp b/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp index 02855ecfbf8..299757e38b3 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp @@ -1,36 +1,36 @@ -#include "schemeshard_impl.h" -#include "schemeshard_path_describer.h" - +#include "schemeshard_impl.h" +#include "schemeshard_path_describer.h" + #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/tx/scheme_board/populator.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TTxInitPopulator : public TTransactionBase<TSchemeShard> { - using TDescription = NSchemeBoard::TTwoPartDescription; - - TMap<TPathId, TDescription> Descriptions; + using TDescription = NSchemeBoard::TTwoPartDescription; + + TMap<TPathId, TDescription> Descriptions; TSideEffects::TPublications DelayedPublications; - + explicit TTxInitPopulator(TSelf *self, TSideEffects::TPublications&& publications) - : TBase(self) + : TBase(self) , DelayedPublications(publications) - { - } - + { + } + TTxType GetTxType() const override { return TXTYPE_INIT_POPULATOR; } - bool Execute(TTransactionContext&, const TActorContext& ctx) override { + bool Execute(TTransactionContext&, const TActorContext& ctx) override { Y_VERIFY(Self->IsShemeShardConfigured()); for (auto item: Self->PathsById) { TPathId pathId = item.first; TPathElement::TPtr pathEl = item.second; - + // KIKIMR-13173 // planned to drop pathes are added to populator // also such pathes (like BSV PQ) can be considered as deleted by path description however they aren't dropped jet @@ -40,44 +40,44 @@ struct TSchemeShard::TTxInitPopulator : public TTransactionBase<TSchemeShard> { // but once they published on the quorum replicas as deleted directly, further publishing is unnecessary // actually they are publishing strongly on the quorum as part of deleting operation // so it's ok to skip migrated pathes is they has beem marked as deleted - continue; - } - + continue; + } + if (!Self->PathIsActive(pathId)) { continue; } - auto result = DescribePath(Self, ctx, pathId); - TDescription description(std::move(result->PreSerializedData), std::move(result->Record)); - Descriptions.emplace(pathId, std::move(description)); - } - - return true; - } - - void Complete(const TActorContext& ctx) override { + auto result = DescribePath(Self, ctx, pathId); + TDescription description(std::move(result->PreSerializedData), std::move(result->Record)); + Descriptions.emplace(pathId, std::move(description)); + } + + return true; + } + + void Complete(const TActorContext& ctx) override { const ui64 tabletId = Self->TabletID(); const auto &domains = *AppData()->DomainsInfo; const ui32 domainId = domains.GetDomainUidByTabletId(tabletId); const ui32 boardSSId = domains.GetDomain(domainId).DefaultSchemeBoardGroup; - IActor* populator = CreateSchemeBoardPopulator( + IActor* populator = CreateSchemeBoardPopulator( tabletId, Self->Generation(), boardSSId, std::move(Descriptions), Self->NextLocalPathId - ); + ); Y_VERIFY(!Self->SchemeBoardPopulator); - Self->SchemeBoardPopulator = Self->Register(populator); - - Self->PublishToSchemeBoard(std::move(DelayedPublications), ctx); - Self->SignalTabletActive(ctx); - } - + Self->SchemeBoardPopulator = Self->Register(populator); + + Self->PublishToSchemeBoard(std::move(DelayedPublications), ctx); + Self->SignalTabletActive(ctx); + } + }; // TSchemeShard::TTxInitPopulator - + NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxInitPopulator(TSideEffects::TPublications&& publications) { return new TTxInitPopulator(this, std::move(publications)); -} - +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp index ed6b709b3e1..74399680cda 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp @@ -1,5 +1,5 @@ #include "schemeshard_impl.h" -#include "schemeshard_path_describer.h" +#include "schemeshard_path_describer.h" #include <ydb/core/tablet/tablet_exception.h> #include <ydb/core/tablet_flat/flat_cxx_database.h> @@ -54,7 +54,7 @@ struct TSchemeShard::TTxInitRoot : public TSchemeShard::TRwTxBase { Self->RootPathElemets = std::move(rootPathElemets); - TSubDomainInfo::TPtr newDomain = new TSubDomainInfo(0, Self->RootPathId()); + TSubDomainInfo::TPtr newDomain = new TSubDomainInfo(0, Self->RootPathId()); newDomain->InitializeAsGlobal(Self->CreateRootProcessingParams(ctx)); Self->SubDomains[Self->RootPathId()] = newDomain; @@ -294,20 +294,20 @@ struct TSchemeShard::TTxInitTenantSchemeShard : public TSchemeShard::TRwTxBase { newPath->CachedEffectiveACL.Update(Self->ParentDomainCachedEffectiveACL, newPath->ACL, newPath->IsContainer()); - TPathId resourcesDomainId = Self->ParentDomainId; - if (record.HasResourcesDomainOwnerId() && record.HasResourcesDomainPathId()) { - resourcesDomainId = TPathId(record.GetResourcesDomainOwnerId(), record.GetResourcesDomainPathId()); - } - + TPathId resourcesDomainId = Self->ParentDomainId; + if (record.HasResourcesDomainOwnerId() && record.HasResourcesDomainPathId()) { + resourcesDomainId = TPathId(record.GetResourcesDomainOwnerId(), record.GetResourcesDomainPathId()); + } + TSubDomainInfo::TPtr subdomain = new TSubDomainInfo(processingParams.GetVersion(), processingParams.GetPlanResolution(), - processingParams.GetTimeCastBucketsPerMediator(), - resourcesDomainId); + processingParams.GetTimeCastBucketsPerMediator(), + resourcesDomainId); + + if (record.HasSharedHive()) { + subdomain->SetSharedHive(TTabletId(record.GetSharedHive())); + } - if (record.HasSharedHive()) { - subdomain->SetSharedHive(TTabletId(record.GetSharedHive())); - } - for (auto& x: storagePools) { subdomain->AddStoragePool(x); } diff --git a/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp b/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp index f5e9f81c611..76e73182496 100644 --- a/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp @@ -1119,7 +1119,7 @@ private: << "State: " << TTxState::StateName(txState.State) << Endl << "MinStep: " << txState.MinStep << Endl << "ReadyForNotifications: " << txState.ReadyForNotifications << Endl - << "DataTotalSize: " << txState.DataTotalSize << Endl + << "DataTotalSize: " << txState.DataTotalSize << Endl << "TxShardsListFinalized: " << txState.TxShardsListFinalized << Endl << "SchemeOpSeqNo: " << txState.SchemeOpSeqNo.Generation << ":" << txState.SchemeOpSeqNo.Round << Endl << "StartTime: " << txState.StartTime << Endl @@ -1217,27 +1217,27 @@ private: } } - H4() {str << "ShardStatuses for opId " << opId << "</a>";} - TABLE_SORTABLE_CLASS("ShardStatuses") { + H4() {str << "ShardStatuses for opId " << opId << "</a>";} + TABLE_SORTABLE_CLASS("ShardStatuses") { TABLEHEAD() { TABLER() { TABLEH() {str << "ShardId";} - TABLEH() {str << "Success";} + TABLEH() {str << "Success";} TABLEH() {str << "Error";} - TABLEH() {str << "BytesProcessed";} - TABLEH() {str << "RowsProcessed";} + TABLEH() {str << "BytesProcessed";} + TABLEH() {str << "RowsProcessed";} } } ui32 maxItems = 10; - for (auto& item: txState.ShardStatuses) { + for (auto& item: txState.ShardStatuses) { if (0 == maxItems) { break; } --maxItems; TABLER() { TABLED() { str << item.first; } - TABLED() { str << item.second.Success; } - TABLED() { str << item.second.Error; } - TABLED() { str << item.second.BytesProcessed; } - TABLED() { str << item.second.RowsProcessed; } + TABLED() { str << item.second.Success; } + TABLED() { str << item.second.Error; } + TABLED() { str << item.second.BytesProcessed; } + TABLED() { str << item.second.RowsProcessed; } } } } diff --git a/ydb/core/tx/schemeshard/schemeshard__notify.cpp b/ydb/core/tx/schemeshard/schemeshard__notify.cpp index 23b8f5d6b32..d390367947a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__notify.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__notify.cpp @@ -23,35 +23,35 @@ struct TSchemeShard::TTxNotifyCompletion : public TSchemeShard::TRwTxBase { if (Self->Operations.contains(TTxId(rawTxId))) { auto txId = TTxId(rawTxId); - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "NotifyTxCompletion" - << " operation in-flight" - << ", txId: " << txId - << ", at schemeshard: " << Self->TabletID()); - - TOperation::TPtr operation = Self->Operations.at(txId); - if (operation->IsReadyToNotify(ctx)) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "NotifyTxCompletion" - << ", operation is ready to notificate" - << ", txId: " << txId - << ", at schemeshard: " << Self->TabletID()); + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "NotifyTxCompletion" + << " operation in-flight" + << ", txId: " << txId + << ", at schemeshard: " << Self->TabletID()); + + TOperation::TPtr operation = Self->Operations.at(txId); + if (operation->IsReadyToNotify(ctx)) { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "NotifyTxCompletion" + << ", operation is ready to notificate" + << ", txId: " << txId + << ", at schemeshard: " << Self->TabletID()); Result = new TEvSchemeShard::TEvNotifyTxCompletionResult(ui64(txId)); - return; - } - - operation->AddNotifySubscriber(Ev->Sender); + return; + } + + operation->AddNotifySubscriber(Ev->Sender); Result = new TEvSchemeShard::TEvNotifyTxCompletionRegistered(ui64(txId)); } else if (Self->Publications.contains(TTxId(rawTxId))) { auto txId = TTxId(rawTxId); - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "NotifyTxCompletion" - << " publication in-flight" + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "NotifyTxCompletion" + << " publication in-flight" << ", count: " << Self->Publications.at(txId).Paths.size() - << ", txId: " << txId - << ", at schemeshard: " << Self->TabletID()); - - Self->Publications.at(txId).Subscribers.insert(Ev->Sender); + << ", txId: " << txId + << ", at schemeshard: " << Self->TabletID()); + + Self->Publications.at(txId).Subscribers.insert(Ev->Sender); Result = new TEvSchemeShard::TEvNotifyTxCompletionRegistered(ui64(txId)); } else if (Self->Exports.contains(rawTxId)) { auto txId = rawTxId; @@ -74,26 +74,26 @@ struct TSchemeShard::TTxNotifyCompletion : public TSchemeShard::TRwTxBase { exportInfo->AddNotifySubscriber(Ev->Sender); Result = new TEvSchemeShard::TEvNotifyTxCompletionRegistered(ui64(txId)); - } else if (Self->Imports.contains(rawTxId)) { - auto txId = rawTxId; - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "NotifyTxCompletion" - << " import in-flight" - << ", txId: " << txId - << ", at schemeshard: " << Self->TabletID()); - - TImportInfo::TPtr importInfo = Self->Imports.at(txId); - if (importInfo->IsFinished()) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "NotifyTxCompletion" - << ", import is ready to notificate" - << ", txId: " << txId - << ", at schemeshard: " << Self->TabletID()); + } else if (Self->Imports.contains(rawTxId)) { + auto txId = rawTxId; + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "NotifyTxCompletion" + << " import in-flight" + << ", txId: " << txId + << ", at schemeshard: " << Self->TabletID()); + + TImportInfo::TPtr importInfo = Self->Imports.at(txId); + if (importInfo->IsFinished()) { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "NotifyTxCompletion" + << ", import is ready to notificate" + << ", txId: " << txId + << ", at schemeshard: " << Self->TabletID()); Result = new TEvSchemeShard::TEvNotifyTxCompletionResult(txId); - return; - } - - importInfo->AddNotifySubscriber(Ev->Sender); + return; + } + + importInfo->AddNotifySubscriber(Ev->Sender); Result = new TEvSchemeShard::TEvNotifyTxCompletionRegistered(ui64(txId)); } else if (Self->IndexBuilds.contains(TIndexBuildId(rawTxId))) { auto txId = TIndexBuildId(rawTxId); @@ -117,7 +117,7 @@ struct TSchemeShard::TTxNotifyCompletion : public TSchemeShard::TRwTxBase { indexInfo->AddNotifySubscriber(Ev->Sender); Result = new TEvSchemeShard::TEvNotifyTxCompletionRegistered(ui64(txId)); - } else { + } else { LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "NotifyTxCompletion" << ", unknown transaction" diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index 0ec8fb11ae1..daf1218bcf7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -10,8 +10,8 @@ #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/tablet_flat/tablet_flat_executor.h> -#include <util/generic/algorithm.h> - +#include <util/generic/algorithm.h> + namespace NKikimr { namespace NSchemeShard { @@ -26,22 +26,22 @@ struct TSchemeShard::TTxOperationProposeCancelTx: public NTabletFlatExecutor::TT TTxOperationProposeCancelTx(TSchemeShard* self, TEvSchemeShard::TEvCancelTx::TPtr ev) : TBase(self) - , Ev(ev) + , Ev(ev) {} - TTxType GetTxType() const override { return TXTYPE_CANCEL_BACKUP_IMPL; } + TTxType GetTxType() const override { return TXTYPE_CANCEL_BACKUP_IMPL; } bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { - const auto& record = Ev->Get()->Record; + const auto& record = Ev->Get()->Record; LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxOperationProposeCancelTx Execute" + "TTxOperationProposeCancelTx Execute" << ", at schemeshard: " << Self->TabletID() << ", message: " << record.ShortDebugString()); txc.DB.NoMoreReadsForTx(); - ISubOperationBase::TPtr part = CreateTxCancelTx(Ev); + ISubOperationBase::TPtr part = CreateTxCancelTx(Ev); TOperationContext context{Self, txc, ctx, OnComplete, MemChanges, DbChanges}; auto fakeResponse = part->Propose(TString(), context); Y_UNUSED(fakeResponse); @@ -53,7 +53,7 @@ struct TSchemeShard::TTxOperationProposeCancelTx: public NTabletFlatExecutor::TT void Complete(const TActorContext& ctx) override { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxOperationProposeCancelTx Complete" + "TTxOperationProposeCancelTx Complete" << ", at schemeshard: " << Self->TabletID()); OnComplete.ApplyOnComplete(Self, ctx); @@ -220,7 +220,7 @@ struct TSchemeShard::TTxOperationPropose: public NTabletFlatExecutor::TTransacti : TBase(self) , Request(request) {} - + TTxType GetTxType() const override { return TXTYPE_PROPOSE; } bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { @@ -230,7 +230,7 @@ struct TSchemeShard::TTxOperationPropose: public NTabletFlatExecutor::TTransacti "TTxOperationPropose Execute" << ", message: " << GetRecordForPrint(Request->Get()->Record).ShortDebugString() << ", at schemeshard: " << selfId); - + txc.DB.NoMoreReadsForTx(); TMemoryChanges memChanges; @@ -458,7 +458,7 @@ struct TSchemeShard::TTxOperationPlanStep: public NTabletFlatExecutor::TTransact NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxOperationPropose(TEvSchemeShard::TEvCancelTx::TPtr& ev) { - return new TTxOperationProposeCancelTx(this, ev); + return new TTxOperationProposeCancelTx(this, ev); } NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxOperationPropose(TEvSchemeShard::TEvModifySchemeTransaction::TPtr& ev) { @@ -481,14 +481,14 @@ NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxOperationProgress(TOper SCHEMESHARD_INCOMING_EVENTS(DefineCreateTxOperationReply) #undef DefineTxOperationReply -TString JoinPath(const TString& workingDir, const TString& name) { - Y_VERIFY(!name.StartsWith('/') && !name.EndsWith('/')); - return TStringBuilder() - << workingDir - << (workingDir.EndsWith('/') ? "" : "/") - << name; -} - +TString JoinPath(const TString& workingDir, const TString& name) { + Y_VERIFY(!name.StartsWith('/') && !name.EndsWith('/')); + return TStringBuilder() + << workingDir + << (workingDir.EndsWith('/') ? "" : "/") + << name; +} + TOperation::TConsumeQuotaResult TOperation::ConsumeQuota(const TTxTransaction& tx, const TOperationContext& context) { TConsumeQuotaResult result; @@ -541,227 +541,227 @@ TOperation::TConsumeQuotaResult TOperation::ConsumeQuota(const TTxTransaction& t TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context) { TSplitTransactionsResult result; - const TPath parentPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); - { - TPath::TChecker checks = parentPath.Check(); - checks + const TPath parentPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); + { + TPath::TChecker checks = parentPath.Check(); + checks .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() .IsLikeDirectory(); - - if (!checks) { + + if (!checks) { result.Transactions.push_back(tx); return result; - } - } - - TString targetName; - - switch (tx.GetOperationType()) { + } + } + + TString targetName; + + switch (tx.GetOperationType()) { case NKikimrSchemeOp::EOperationType::ESchemeOpMkDir: - targetName = tx.GetMkDir().GetName(); - break; + targetName = tx.GetMkDir().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable: - if (tx.GetCreateTable().HasCopyFromTable()) { + if (tx.GetCreateTable().HasCopyFromTable()) { result.Transactions.push_back(tx); return result; - } - targetName = tx.GetCreateTable().GetName(); - break; + } + targetName = tx.GetCreateTable().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup: - targetName = tx.GetCreatePersQueueGroup().GetName(); - break; + targetName = tx.GetCreatePersQueueGroup().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain: case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExtSubDomain: - targetName = tx.GetSubDomain().GetName(); - break; + targetName = tx.GetSubDomain().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateRtmrVolume: - targetName = tx.GetCreateRtmrVolume().GetName(); - break; + targetName = tx.GetCreateRtmrVolume().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateBlockStoreVolume: - targetName = tx.GetCreateBlockStoreVolume().GetName(); - break; + targetName = tx.GetCreateBlockStoreVolume().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateFileStore: targetName = tx.GetCreateFileStore().GetName(); break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateKesus: - targetName = tx.GetKesus().GetName(); - break; + targetName = tx.GetKesus().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSolomonVolume: - targetName = tx.GetCreateSolomonVolume().GetName(); - break; + targetName = tx.GetCreateSolomonVolume().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable: - targetName = tx.GetCreateIndexedTable().GetTableDescription().GetName(); - break; + targetName = tx.GetCreateIndexedTable().GetTableDescription().GetName(); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnStore: targetName = tx.GetCreateColumnStore().GetName(); break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable: targetName = tx.GetCreateColumnTable().GetName(); break; - default: + default: result.Transactions.push_back(tx); return result; - } - - if (!targetName || targetName.StartsWith('/') || targetName.EndsWith('/')) { + } + + if (!targetName || targetName.StartsWith('/') || targetName.EndsWith('/')) { result.Transactions.push_back(tx); return result; - } - - TPath path = TPath::Resolve(JoinPath(tx.GetWorkingDir(), targetName), context.SS); - { - TPath::TChecker checks = path.Check(); - checks.IsAtLocalSchemeShard(); - - bool exists = false; - if (path.IsResolved()) { - checks.IsResolved(); - exists = !path.IsDeleted(); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (checks && !exists) { - checks - .IsValidLeafName() - .DepthLimit() - .PathsLimit(); - } - - if (checks && !exists && path.Parent().IsResolved()) { - checks.DirChildrenLimit(); - } - - if (!checks) { + } + + TPath path = TPath::Resolve(JoinPath(tx.GetWorkingDir(), targetName), context.SS); + { + TPath::TChecker checks = path.Check(); + checks.IsAtLocalSchemeShard(); + + bool exists = false; + if (path.IsResolved()) { + checks.IsResolved(); + exists = !path.IsDeleted(); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks && !exists) { + checks + .IsValidLeafName() + .DepthLimit() + .PathsLimit(); + } + + if (checks && !exists && path.Parent().IsResolved()) { + checks.DirChildrenLimit(); + } + + if (!checks) { result.Status = checks.GetStatus(&result.Reason); result.Transactions.push_back(tx); return result; - } - - const TString name = path.LeafName(); - path.Rise(); - - TTxTransaction create(tx); - create.SetWorkingDir(path.PathString()); + } + + const TString name = path.LeafName(); + path.Rise(); + + TTxTransaction create(tx); + create.SetWorkingDir(path.PathString()); create.SetFailOnExist(tx.GetFailOnExist()); - - switch (tx.GetOperationType()) { + + switch (tx.GetOperationType()) { case NKikimrSchemeOp::EOperationType::ESchemeOpMkDir: - create.MutableMkDir()->SetName(name); - break; + create.MutableMkDir()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable: - create.MutableCreateTable()->SetName(name); - break; + create.MutableCreateTable()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup: - create.MutableCreatePersQueueGroup()->SetName(name); - break; + create.MutableCreatePersQueueGroup()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain: case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExtSubDomain: - create.MutableSubDomain()->SetName(name); - break; + create.MutableSubDomain()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateRtmrVolume: - create.MutableCreateRtmrVolume()->SetName(name); - break; + create.MutableCreateRtmrVolume()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateBlockStoreVolume: - create.MutableCreateBlockStoreVolume()->SetName(name); - break; + create.MutableCreateBlockStoreVolume()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateFileStore: create.MutableCreateFileStore()->SetName(name); break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateKesus: - create.MutableKesus()->SetName(name); - break; + create.MutableKesus()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSolomonVolume: - create.MutableCreateSolomonVolume()->SetName(name); - break; + create.MutableCreateSolomonVolume()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable: - create.MutableCreateIndexedTable()->MutableTableDescription()->SetName(name); - break; + create.MutableCreateIndexedTable()->MutableTableDescription()->SetName(name); + break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnStore: create.MutableCreateColumnStore()->SetName(name); break; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable: create.MutableCreateColumnTable()->SetName(name); break; - default: - Y_UNREACHABLE(); - } - + default: + Y_UNREACHABLE(); + } + result.Transactions.push_back(create); - - if (exists) { - return result; - } - } - - while (path != parentPath) { - TPath::TChecker checks = path.Check(); + + if (exists) { + return result; + } + } + + while (path != parentPath) { + TPath::TChecker checks = path.Check(); checks .NotUnderDomainUpgrade() .IsAtLocalSchemeShard(); - - if (path.IsResolved()) { - checks.IsResolved(); - - if (path.IsDeleted()) { - checks.IsDeleted(); - } else { - checks - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() + + if (path.IsResolved()) { + checks.IsResolved(); + + if (path.IsDeleted()) { + checks.IsDeleted(); + } else { + checks + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() .IsLikeDirectory(); - - if (checks) { - break; - } - } - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (checks) { - checks - .IsValidLeafName() - .DepthLimit() + + if (checks) { + break; + } + } + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .DepthLimit() .PathsLimit(result.Transactions.size() + 1); - } - - if (checks && path.Parent().IsResolved()) { - checks.DirChildrenLimit(); - } - - if (!checks) { + } + + if (checks && path.Parent().IsResolved()) { + checks.DirChildrenLimit(); + } + + if (!checks) { result.Status = checks.GetStatus(&result.Reason); result.Transactions.clear(); result.Transactions.push_back(tx); return result; - } - - const TString name = path.LeafName(); - path.Rise(); - - TTxTransaction mkdir; + } + + const TString name = path.LeafName(); + path.Rise(); + + TTxTransaction mkdir; mkdir.SetFailOnExist(true); mkdir.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpMkDir); - mkdir.SetWorkingDir(path.PathString()); - mkdir.MutableMkDir()->SetName(name); + mkdir.SetWorkingDir(path.PathString()); + mkdir.MutableMkDir()->SetName(name); result.Transactions.push_back(mkdir); - } - + } + Reverse(result.Transactions.begin(), result.Transactions.end()); - return result; -} - + return result; +} + ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxState::ETxState txState) { switch (txType) { case TTxState::ETxType::TxMkDir: @@ -782,9 +782,9 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta case TTxState::ETxType::TxMergeTablePartition: return CreateSplitMerge(NextPartId(), txState); case TTxState::ETxType::TxBackup: - return CreateBackup(NextPartId(), txState); - case TTxState::ETxType::TxRestore: - return CreateRestore(NextPartId(), txState); + return CreateBackup(NextPartId(), txState); + case TTxState::ETxType::TxRestore: + return CreateRestore(NextPartId(), txState); case TTxState::ETxType::TxDropTable: return CreateDropTable(NextPartId(), txState); case TTxState::ETxType::TxCreateTableIndex: @@ -872,20 +872,20 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta case TTxState::ETxType::TxDropFileStore: return CreateDropFileStore(NextPartId(), txState); - // CDC - case TTxState::ETxType::TxCreateCdcStream: - return CreateNewCdcStreamImpl(NextPartId(), txState); - case TTxState::ETxType::TxCreateCdcStreamAtTable: - return CreateNewCdcStreamAtTable(NextPartId(), txState); - case TTxState::ETxType::TxAlterCdcStream: - return CreateAlterCdcStreamImpl(NextPartId(), txState); - case TTxState::ETxType::TxAlterCdcStreamAtTable: - return CreateAlterCdcStreamAtTable(NextPartId(), txState); - case TTxState::ETxType::TxDropCdcStream: - return CreateDropCdcStreamImpl(NextPartId(), txState); - case TTxState::ETxType::TxDropCdcStreamAtTable: - return CreateDropCdcStreamAtTable(NextPartId(), txState); - + // CDC + case TTxState::ETxType::TxCreateCdcStream: + return CreateNewCdcStreamImpl(NextPartId(), txState); + case TTxState::ETxType::TxCreateCdcStreamAtTable: + return CreateNewCdcStreamAtTable(NextPartId(), txState); + case TTxState::ETxType::TxAlterCdcStream: + return CreateAlterCdcStreamImpl(NextPartId(), txState); + case TTxState::ETxType::TxAlterCdcStreamAtTable: + return CreateAlterCdcStreamAtTable(NextPartId(), txState); + case TTxState::ETxType::TxDropCdcStream: + return CreateDropCdcStreamImpl(NextPartId(), txState); + case TTxState::ETxType::TxDropCdcStreamAtTable: + return CreateDropCdcStreamAtTable(NextPartId(), txState); + // Sequences case TTxState::ETxType::TxCreateSequence: return CreateNewSequence(NextPartId(), txState); @@ -902,14 +902,14 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta case TTxState::ETxType::TxMoveTableIndex: return CreateMoveTableIndex(NextPartId(), txState); - // Replication - case TTxState::ETxType::TxCreateReplication: - return CreateNewReplication(NextPartId(), txState); - case TTxState::ETxType::TxAlterReplication: - Y_FAIL("TODO: implement"); - case TTxState::ETxType::TxDropReplication: - return CreateDropReplication(NextPartId(), txState); - + // Replication + case TTxState::ETxType::TxCreateReplication: + return CreateNewReplication(NextPartId(), txState); + case TTxState::ETxType::TxAlterReplication: + Y_FAIL("TODO: implement"); + case TTxState::ETxType::TxDropReplication: + return CreateDropReplication(NextPartId(), txState); + case TTxState::ETxType::TxInvalid: Y_UNREACHABLE(); } @@ -920,37 +920,37 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationType opType, const TTxTransaction& tx) { switch (opType) { case NKikimrSchemeOp::EOperationType::ESchemeOpMkDir: - return CreateMkDir(NextPartId(), tx); + return CreateMkDir(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpRmDir: - return CreateRmDir(NextPartId(), tx); + return CreateRmDir(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpModifyACL: - return CreateModifyACL(NextPartId(), tx); + return CreateModifyACL(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes: - return CreateAlterUserAttrs(NextPartId(), tx); + return CreateAlterUserAttrs(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpForceDropUnsafe: - return CreateFroceDropUnsafe(NextPartId(), tx); + return CreateFroceDropUnsafe(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable: return CreateNewTable(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterTable: - return CreateAlterTable(NextPartId(), tx); + return CreateAlterTable(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpSplitMergeTablePartitions: - return CreateSplitMerge(NextPartId(), tx); + return CreateSplitMerge(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpBackup: - return CreateBackup(NextPartId(), tx); + return CreateBackup(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpRestore: - return CreateRestore(NextPartId(), tx); + return CreateRestore(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropTable: Y_FAIL("in general, drop table is multipart operation now due table indexes"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTableIndex: Y_FAIL("is handled as part of ESchemeOpCreateIndexedTable"); case NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex: Y_FAIL("is handled as part of ESchemeOpDropTable"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateRtmrVolume: - return CreateNewRTMR(NextPartId(), tx); + return CreateNewRTMR(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnStore: return CreateNewOlapStore(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterColumnStore: @@ -964,43 +964,43 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp case NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnTable: return CreateDropOlapTable(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup: - return CreateNewPQ(NextPartId(), tx); + return CreateNewPQ(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup: - return CreateAlterPQ(NextPartId(), tx); + return CreateAlterPQ(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup: - return CreateDropPQ(NextPartId(), tx); + return CreateDropPQ(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSolomonVolume: - return CreateNewSolomon(NextPartId(), tx); - case NKikimrSchemeOp::EOperationType::ESchemeOpAlterSolomonVolume: - return CreateAlterSolomon(NextPartId(), tx); + return CreateNewSolomon(NextPartId(), tx); + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterSolomonVolume: + return CreateAlterSolomon(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropSolomonVolume: - return CreateDropSolomon(NextPartId(), tx); + return CreateDropSolomon(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain: - return CreateSubDomain(NextPartId(), tx); + return CreateSubDomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterSubDomain: Y_FAIL("run in compatible"); case NKikimrSchemeOp::EOperationType::ESchemeOpDropSubDomain: - return CreateDropSubdomain(NextPartId(), tx); + return CreateDropSubdomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpForceDropSubDomain: Y_FAIL("run in compatible"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExtSubDomain: - return CreateExtSubDomain(NextPartId(), tx); + return CreateExtSubDomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExtSubDomain: - return CreateAlterExtSubDomain(NextPartId(), tx); + return CreateAlterExtSubDomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpForceDropExtSubDomain: return CreateFroceDropExtSubDomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateKesus: - return CreateNewKesus(NextPartId(), tx); + return CreateNewKesus(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterKesus: - return CreateAlterKesus(NextPartId(), tx); + return CreateAlterKesus(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropKesus: - return CreateDropKesus(NextPartId(), tx); + return CreateDropKesus(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpUpgradeSubDomain: return CreateUpgradeSubDomain(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpUpgradeSubDomainDecision: return CreateUpgradeSubDomainDecision(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexBuild: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLockForIndexBuild: return CreateLockForIndexBuild(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropLock: @@ -1036,60 +1036,60 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp case NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence: return CreateDropSequence(NextPartId(), tx); - // Index + // Index case NKikimrSchemeOp::EOperationType::ESchemeOpApplyIndexBuild: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterTableIndex: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpFinalizeBuildIndexImplTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpFinalizeBuildIndexMainTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpCancelIndexBuild: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpDropIndex: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndexAtMainTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); - // CDC + // CDC case NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream: case NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStreamImpl: case NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStreamAtTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStream: case NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStreamImpl: case NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStreamAtTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); + Y_FAIL("multipart operations are handled before, also they require transaction details"); case NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStream: case NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamImpl: case NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamAtTable: - Y_FAIL("multipart operations are handled before, also they require transaction details"); - + Y_FAIL("multipart operations are handled before, also they require transaction details"); + case NKikimrSchemeOp::EOperationType::ESchemeOp_DEPRECATED_35: Y_FAIL("imposible"); - // Move + // Move case NKikimrSchemeOp::EOperationType::ESchemeOpMoveTable: return CreateMoveTable(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpMoveTableIndex: return CreateMoveTableIndex(NextPartId(), tx); - - // Replication - case NKikimrSchemeOp::EOperationType::ESchemeOpCreateReplication: - return CreateNewReplication(NextPartId(), tx); - case NKikimrSchemeOp::EOperationType::ESchemeOpAlterReplication: - Y_FAIL("TODO: implement"); - case NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication: - return CreateDropReplication(NextPartId(), tx); + + // Replication + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateReplication: + return CreateNewReplication(NextPartId(), tx); + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterReplication: + Y_FAIL("TODO: implement"); + case NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication: + return CreateDropReplication(NextPartId(), tx); } Y_UNREACHABLE(); @@ -1098,12 +1098,12 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp TVector<ISubOperationBase::TPtr> TOperation::ConstructParts(const TTxTransaction& tx, TOperationContext& context) { const auto& opType = tx.GetOperationType(); - switch (opType) { + switch (opType) { case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable: if (tx.GetCreateTable().HasCopyFromTable()) { return {CreateCopyTable(NextPartId(), tx, context)}; // Copy indexes table as well as common table } - return {ConstructPart(opType, tx)}; + return {ConstructPart(opType, tx)}; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable: return CreateIndexedTable(NextPartId(), tx, context); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables: @@ -1123,27 +1123,27 @@ TVector<ISubOperationBase::TPtr> TOperation::ConstructParts(const TTxTransaction case NKikimrSchemeOp::EOperationType::ESchemeOpAlterSubDomain: return {CreateCompatibleSubdomainAlter(context.SS, NextPartId(), tx)}; case NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream: - return CreateNewCdcStream(NextPartId(), tx, context); + return CreateNewCdcStream(NextPartId(), tx, context); case NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStream: - return CreateAlterCdcStream(NextPartId(), tx, context); + return CreateAlterCdcStream(NextPartId(), tx, context); case NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStream: - return CreateDropCdcStream(NextPartId(), tx, context); + return CreateDropCdcStream(NextPartId(), tx, context); case NKikimrSchemeOp::EOperationType::ESchemeOpMoveTable: return CreateConsistentMoveTable(NextPartId(), tx, context); - default: - return {ConstructPart(opType, tx)}; + default: + return {ConstructPart(opType, tx)}; } } -bool TOperation::AddPublishingPath(TPathId pathId, ui64 version) { - Y_VERIFY(!IsReadyToNotify()); - return Publications.emplace(pathId, version).second; -} - -bool TOperation::IsPublished() const { - return Publications.empty(); -} - +bool TOperation::AddPublishingPath(TPathId pathId, ui64 version) { + Y_VERIFY(!IsReadyToNotify()); + return Publications.emplace(pathId, version).second; +} + +bool TOperation::IsPublished() const { + return Publications.empty(); +} + void TOperation::ReadyToNotifyPart(TSubTxId partId) { ReadyToNotifyParts.insert(partId); } @@ -1152,14 +1152,14 @@ bool TOperation::IsReadyToNotify(const TActorContext& ctx) const { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TOperation IsReadyToNotify" << ", TxId: " << TxId - << ", ready parts: " << ReadyToNotifyParts.size() << "/" << Parts.size() - << ", is published: " << (IsPublished() ? "true" : "false")); + << ", ready parts: " << ReadyToNotifyParts.size() << "/" << Parts.size() + << ", is published: " << (IsPublished() ? "true" : "false")); return IsReadyToNotify(); } bool TOperation::IsReadyToNotify() const { - return IsPublished() && ReadyToNotifyParts.size() == Parts.size(); + return IsPublished() && ReadyToNotifyParts.size() == Parts.size(); } void TOperation::AddNotifySubscriber(const TActorId& actorId) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.h b/ydb/core/tx/schemeshard/schemeshard__operation.h index d9e7c4ea388..52f1848aa12 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation.h @@ -31,9 +31,9 @@ struct TOperation: TSimpleRefCount<TOperation> { using TProposeShards = std::tuple<TSubTxId, TTabletId>; TDeque<TProposeShards> ShardsProposes; - using TPublishPath = std::pair<TPathId, ui64>; - TSet<TPublishPath> Publications; - + using TPublishPath = std::pair<TPathId, ui64>; + TSet<TPublishPath> Publications; + THashSet<TSubTxId> ReadyToNotifyParts; THashSet<TSubTxId> DoneParts; THashMap<TPathId, NKikimrSchemeOp::EPathState> ReleasePathAtDone; @@ -67,16 +67,16 @@ struct TOperation: TSimpleRefCount<TOperation> { static TConsumeQuotaResult ConsumeQuota(const TTxTransaction& tx, const TOperationContext& context); static TSplitTransactionsResult SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context); - + ISubOperationBase::TPtr RestorePart(TTxState::ETxType opType, TTxState::ETxState opState); ISubOperationBase::TPtr ConstructPart(NKikimrSchemeOp::EOperationType opType, const TTxTransaction& tx); TVector<ISubOperationBase::TPtr> ConstructParts(const TTxTransaction& tx, TOperationContext& context); void AddPart(ISubOperationBase::TPtr part) { Parts.push_back(part);} - bool AddPublishingPath(TPathId pathId, ui64 version); - bool IsPublished() const; - + bool AddPublishingPath(TPathId pathId, ui64 version); + bool IsPublished() const; + void ReadyToNotifyPart(TSubTxId partId); bool IsReadyToNotify(const TActorContext& ctx) const; bool IsReadyToNotify() const; @@ -136,9 +136,9 @@ private: TOperationId NextPartId() { return TOperationId(TxId, TSubTxId(Parts.size())); } }; -inline TOperationId NextPartId(const TOperationId& opId, const TVector<ISubOperationBase::TPtr>& parts) { - return TOperationId(opId.GetTxId(), opId.GetSubTxId() + parts.size()); +inline TOperationId NextPartId(const TOperationId& opId, const TVector<ISubOperationBase::TPtr>& parts) { + return TOperationId(opId.GetTxId(), opId.GetSubTxId() + parts.size()); +} + } - } -} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_bsv.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_bsv.cpp index 54407b3f3d4..27455e46270 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_bsv.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_bsv.cpp @@ -12,8 +12,8 @@ using namespace NKikimr; using namespace NSchemeShard; class TAlterBlockStoreVolume: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -27,8 +27,8 @@ class TAlterBlockStoreVolume: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; // DONE ??? - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -44,7 +44,7 @@ class TAlterBlockStoreVolume: public TSubOperation { return THolder(new NBSVState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NBSVState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -61,11 +61,11 @@ class TAlterBlockStoreVolume: public TSubOperation { } public: - TAlterBlockStoreVolume(TOperationId id, const TTxTransaction& tx) + TAlterBlockStoreVolume(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterBlockStoreVolume(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -404,10 +404,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& alter = Transaction.GetAlterBlockStoreVolume(); + const auto& alter = Transaction.GetAlterBlockStoreVolume(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = alter.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = alter.GetName(); const TPathId pathId = alter.HasPathId() ? context.SS->MakeLocalId(alter.GetPathId()) : InvalidPathId; @@ -438,8 +438,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -559,7 +559,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -648,16 +648,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterBlockStoreVolume"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterBlockStoreVolume"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterBlockStoreVolume AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -668,12 +668,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, const TTxTransaction& tx) { - return new TAlterBlockStoreVolume(id, tx); -} - +ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, const TTxTransaction& tx) { + return new TAlterBlockStoreVolume(id, tx); +} + ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterBlockStoreVolume(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp index ee3a632b48b..d7027b570e6 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp @@ -1,546 +1,546 @@ -#include "schemeshard__operation_part.h" -#include "schemeshard__operation_common.h" -#include "schemeshard_impl.h" - -#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) - -namespace NKikimr { +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr { namespace NSchemeShard { - -namespace { - -class TPropose: public TSubOperationState { + +namespace { + +class TPropose: public TSubOperationState { TString DebugHint() const override { - return TStringBuilder() - << "AlterCdcStream TPropose" - << " opId# " << OperationId << " "; - } - -public: - explicit TPropose(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxAlterCdcStream); - - // TODO(KIKIMR-12278): shards - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - const auto step = TStepId(ev->Get()->StepId); - - LOG_I(DebugHint() << "HandleReply TEvOperationPlan" - << ": step# " << step); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxAlterCdcStream); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->CdcStreams.contains(pathId)); - auto stream = context.SS->CdcStreams.at(pathId); - - NIceDb::TNiceDb db(context.Txc.DB); - - context.SS->PersistCdcStream(db, pathId); - context.SS->CdcStreams[pathId] = stream->AlterData; - - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - return true; - } - -private: - const TOperationId OperationId; - -}; // TPropose - -class TAlterCdcStream: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::Propose; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return TTxState::Done; - default: - return TTxState::Invalid; - } - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return THolder(new TPropose(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TAlterCdcStream(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TAlterCdcStream(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + return TStringBuilder() + << "AlterCdcStream TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxAlterCdcStream); + + // TODO(KIKIMR-12278): shards + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxAlterCdcStream); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->CdcStreams.contains(pathId)); + auto stream = context.SS->CdcStreams.at(pathId); + + NIceDb::TNiceDb db(context.Txc.DB); + + context.SS->PersistCdcStream(db, pathId); + context.SS->CdcStreams[pathId] = stream->AlterData; + + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TAlterCdcStream: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return THolder(new TPropose(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TAlterCdcStream(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TAlterCdcStream(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetAlterCdcStream(); - const auto& streamName = op.GetStreamName(); - - LOG_N("TAlterCdcStream Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << streamName); - + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetAlterCdcStream(); + const auto& streamName = op.GetStreamName(); + + LOG_N("TAlterCdcStream Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto streamPath = TPath::Resolve(workingDir, context.SS).Dive(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .NotUnderDeleting(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - const auto tablePath = streamPath.Parent(); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - context.DbChanges.PersistAlterCdcStream(streamPath.Base()->PathId); - context.DbChanges.PersistTxState(OperationId); - - Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); - auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); - - auto streamAlter = stream->CreateNextVersion(); - Y_VERIFY(streamAlter); - - switch (op.GetActionCase()) { + + const auto streamPath = TPath::Resolve(workingDir, context.SS).Dive(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .NotUnderDeleting(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + const auto tablePath = streamPath.Parent(); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + context.DbChanges.PersistAlterCdcStream(streamPath.Base()->PathId); + context.DbChanges.PersistTxState(OperationId); + + Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); + auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); + + auto streamAlter = stream->CreateNextVersion(); + Y_VERIFY(streamAlter); + + switch (op.GetActionCase()) { case NKikimrSchemeOp::TAlterCdcStream::kDisable: - streamAlter->State = TCdcStreamInfo::EState::ECdcStreamStateDisabled; - break; - default: + streamAlter->State = TCdcStreamInfo::EState::ECdcStreamStateDisabled; + break; + default: result->SetError(NKikimrScheme::StatusInvalidParameter, TStringBuilder() - << "Unknown action: " << static_cast<ui32>(op.GetActionCase())); - return result; - } - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterCdcStream, streamPath.Base()->PathId); - txState.State = TTxState::Propose; - txState.MinStep = TStepId(1); - - streamPath.Base()->PathState = TPathElement::EPathState::EPathStateAlter; - streamPath.Base()->LastTxId = OperationId.GetTxId(); - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterCdcStream"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TAlterCdcStream AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TAlterCdcStream - -class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { -protected: - void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - auto table = context.SS->Tables.at(pathId); - - auto& notice = *tx.MutableAlterCdcStreamNotice(); - PathIdFromPathId(pathId, notice.MutablePathId()); - notice.SetTableSchemaVersion(table->AlterVersion + 1); - - bool found = false; - for (const auto& [childName, childPathId] : path->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsCdcStream() || childPath->Dropped() || childPath->NormalState()) { - continue; - } - - Y_VERIFY(context.SS->CdcStreams.contains(childPathId)); - auto stream = context.SS->CdcStreams.at(childPathId); - - Y_VERIFY_S(!found, "Too many cdc streams are planned to alter" - << ": found# " << PathIdFromPathId(notice.GetStreamDescription().GetPathId()) - << ", another# " << childPathId); - found = true; - - Y_VERIFY(stream->AlterData); - context.SS->DescribeCdcStream(childPathId, childName, stream->AlterData, *notice.MutableStreamDescription()); - } - } - -public: - using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; - -}; // TConfigurePartsAtTable - -class TAlterCdcStreamAtTable: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::ConfigureParts; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return TTxState::Propose; - case TTxState::Propose: - return TTxState::ProposedWaitParts; - case TTxState::ProposedWaitParts: - return TTxState::Done; - default: - return TTxState::Invalid; - } - - return TTxState::Invalid; - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return THolder(new TConfigurePartsAtTable(OperationId)); - case TTxState::Propose: - return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); - case TTxState::ProposedWaitParts: - return THolder(new NTableState::TProposedWaitParts(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + << "Unknown action: " << static_cast<ui32>(op.GetActionCase())); + return result; + } + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterCdcStream, streamPath.Base()->PathId); + txState.State = TTxState::Propose; + txState.MinStep = TStepId(1); + + streamPath.Base()->PathState = TPathElement::EPathState::EPathStateAlter; + streamPath.Base()->LastTxId = OperationId.GetTxId(); + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterCdcStream"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TAlterCdcStream AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TAlterCdcStream + +class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { +protected: + void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + auto table = context.SS->Tables.at(pathId); + + auto& notice = *tx.MutableAlterCdcStreamNotice(); + PathIdFromPathId(pathId, notice.MutablePathId()); + notice.SetTableSchemaVersion(table->AlterVersion + 1); + + bool found = false; + for (const auto& [childName, childPathId] : path->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsCdcStream() || childPath->Dropped() || childPath->NormalState()) { + continue; + } + + Y_VERIFY(context.SS->CdcStreams.contains(childPathId)); + auto stream = context.SS->CdcStreams.at(childPathId); + + Y_VERIFY_S(!found, "Too many cdc streams are planned to alter" + << ": found# " << PathIdFromPathId(notice.GetStreamDescription().GetPathId()) + << ", another# " << childPathId); + found = true; + + Y_VERIFY(stream->AlterData); + context.SS->DescribeCdcStream(childPathId, childName, stream->AlterData, *notice.MutableStreamDescription()); + } + } + +public: + using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; + +}; // TConfigurePartsAtTable + +class TAlterCdcStreamAtTable: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::ConfigureParts; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return TTxState::Propose; + case TTxState::Propose: + return TTxState::ProposedWaitParts; + case TTxState::ProposedWaitParts: + return TTxState::Done; + default: + return TTxState::Invalid; + } + + return TTxState::Invalid; + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return THolder(new TConfigurePartsAtTable(OperationId)); + case TTxState::Propose: + return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); + case TTxState::ProposedWaitParts: + return THolder(new NTableState::TProposedWaitParts(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetAlterCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamName = op.GetStreamName(); - - LOG_N("TAlterCdcStreamAtTable Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << tableName << "/" << streamName); - + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetAlterCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamName = op.GetStreamName(); + + LOG_N("TAlterCdcStreamAtTable Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << tableName << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto tablePath = TPath::Resolve(workingDir, context.SS).Dive(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - const auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .IsUnderOperation() - .IsUnderTheSameOperation(OperationId.GetTxId()); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + + const auto tablePath = TPath::Resolve(workingDir, context.SS).Dive(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + const auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .IsUnderOperation() + .IsUnderTheSameOperation(OperationId.GetTxId()); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); - return result; - } - - if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { + return result; + } + + if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); - return result; - } - - context.DbChanges.PersistTxState(OperationId); - - Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); - auto table = context.SS->Tables.at(tablePath.Base()->PathId); - - Y_VERIFY(table->AlterVersion != 0); - Y_VERIFY(!table->AlterData); - - Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); - auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); - - Y_VERIFY(stream->AlterVersion != 0); - Y_VERIFY(stream->AlterData); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterCdcStreamAtTable, tablePath.Base()->PathId); - txState.State = TTxState::ConfigureParts; - + return result; + } + + context.DbChanges.PersistTxState(OperationId); + + Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); + auto table = context.SS->Tables.at(tablePath.Base()->PathId); + + Y_VERIFY(table->AlterVersion != 0); + Y_VERIFY(!table->AlterData); + + Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); + auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); + + Y_VERIFY(stream->AlterVersion != 0); + Y_VERIFY(stream->AlterData); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterCdcStreamAtTable, tablePath.Base()->PathId); + txState.State = TTxState::ConfigureParts; + tablePath.Base()->PathState = NKikimrSchemeOp::EPathStateAlter; - tablePath.Base()->LastTxId = OperationId.GetTxId(); - - for (const auto& splitOpId : table->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); - } - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterCdcStreamAtTable"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TAlterCdcStreamAtTable AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TAlterCdcStreamAtTable - -} // anonymous - -ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { - return new TAlterCdcStream(id, tx); -} - -ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, TTxState::ETxState state) { - return new TAlterCdcStream(id, state); -} - -ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { - return new TAlterCdcStreamAtTable(id, tx); -} - -ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { - return new TAlterCdcStreamAtTable(id, state); -} - -TVector<ISubOperationBase::TPtr> CreateAlterCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { + tablePath.Base()->LastTxId = OperationId.GetTxId(); + + for (const auto& splitOpId : table->GetSplitOpsInFlight()) { + context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); + } + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterCdcStreamAtTable"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TAlterCdcStreamAtTable AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TAlterCdcStreamAtTable + +} // anonymous + +ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { + return new TAlterCdcStream(id, tx); +} + +ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, TTxState::ETxState state) { + return new TAlterCdcStream(id, state); +} + +ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { + return new TAlterCdcStreamAtTable(id, tx); +} + +ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { + return new TAlterCdcStreamAtTable(id, state); +} + +TVector<ISubOperationBase::TPtr> CreateAlterCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStream); - - LOG_D("CreateAlterCdcStream" - << ": opId# " << opId - << ", tx# " << tx.ShortDebugString()); - - const auto& op = tx.GetAlterCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamName = op.GetStreamName(); - - const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); - - const auto tablePath = workingDirPath.Child(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - const auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(tx, errStr)) { + + LOG_D("CreateAlterCdcStream" + << ": opId# " << opId + << ", tx# " << tx.ShortDebugString()); + + const auto& op = tx.GetAlterCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamName = op.GetStreamName(); + + const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); + + const auto tablePath = workingDirPath.Child(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + const auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusPreconditionFailed, errStr)}; - } - - if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { + } + + if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusMultipleModifications, errStr)}; - } - - TVector<ISubOperationBase::TPtr> result; - - { + } + + TVector<ISubOperationBase::TPtr> result; + + { auto outTx = TransactionTemplate(tablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStreamImpl); - outTx.MutableAlterCdcStream()->CopyFrom(op); - - result.push_back(CreateAlterCdcStreamImpl(NextPartId(opId, result), outTx)); - } - - { + outTx.MutableAlterCdcStream()->CopyFrom(op); + + result.push_back(CreateAlterCdcStreamImpl(NextPartId(opId, result), outTx)); + } + + { auto outTx = TransactionTemplate(workingDirPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStreamAtTable); - outTx.MutableAlterCdcStream()->CopyFrom(op); - - result.push_back(CreateAlterCdcStreamAtTable(NextPartId(opId, result), outTx)); - } - - return result; -} - + outTx.MutableAlterCdcStream()->CopyFrom(op); + + result.push_back(CreateAlterCdcStreamAtTable(NextPartId(opId, result), outTx)); + } + + return result; +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp index 87d56b5931f..463f3bb3cf9 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp @@ -37,8 +37,8 @@ void PersistShards(NIceDb::TNiceDb& db, TTxState& txState, ui64 shardsToCreate, } class TAlterExtSubDomain: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -52,8 +52,8 @@ class TAlterExtSubDomain: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; // DONE ??? - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -69,7 +69,7 @@ class TAlterExtSubDomain: public TSubOperation { return THolder(new NSubDomainState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NSubDomainState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -86,11 +86,11 @@ class TAlterExtSubDomain: public TSubOperation { } public: - TAlterExtSubDomain(TOperationId id, const TTxTransaction& tx) + TAlterExtSubDomain(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterExtSubDomain(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -102,10 +102,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& settings = Transaction.GetSubDomain(); + const auto& settings = Transaction.GetSubDomain(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = settings.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = settings.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterExtSubDomain Propose" @@ -133,8 +133,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -377,8 +377,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -412,7 +412,7 @@ public: if (addExternalHive) { DeclareShards(txState, OperationId.GetTxId(), subDomain->PathId, 1, TTabletTypes::Hive, channelBindings, context.SS); - } else if (!alterData->GetSharedHive()) { + } else if (!alterData->GetSharedHive()) { alterData->SetSharedHive(context.SS->GetGlobalHive(context.Ctx)); } @@ -441,16 +441,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterSubDomain"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterSubDomain"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterSubDomain AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -461,12 +461,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, const TTxTransaction& tx) { - return new TAlterExtSubDomain(id, tx); -} - +ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, const TTxTransaction& tx) { + return new TAlterExtSubDomain(id, tx); +} + ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterExtSubDomain(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_kesus.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_kesus.cpp index 4d2fe264fae..38e5f747774 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_kesus.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_kesus.cpp @@ -190,8 +190,8 @@ public: context.SS->PersistRemoveKesusAlter(db, pathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.OnComplete.DoneOperation(OperationId); return true; } @@ -215,8 +215,8 @@ public: class TAlterKesus: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -256,11 +256,11 @@ class TAlterKesus: public TSubOperation { } public: - TAlterKesus(TOperationId id, const TTxTransaction& tx) + TAlterKesus(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterKesus(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -293,10 +293,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& alter = Transaction.GetKesus(); + const auto& alter = Transaction.GetKesus(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = alter.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = alter.GetName(); const TPathId pathId = alter.HasPathId() ? context.SS->MakeLocalId(alter.GetPathId()) : InvalidPathId; LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -321,8 +321,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -357,7 +357,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -376,16 +376,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterKesus"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterKesus"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterKesus AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -396,12 +396,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, const TTxTransaction& tx) { - return new TAlterKesus(id, tx); -} - +ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, const TTxTransaction& tx) { + return new TAlterKesus(id, tx); +} + ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterKesus(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp index ab08cbd82f6..2e89cad9869 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp @@ -539,7 +539,7 @@ public: } NKikimrScheme::EStatus status; - TOlapTableInfo::TPtr alterData = ParseParams(path, tableInfo, storeInfo, alter, *path.DomainInfo(), status, errStr, context); + TOlapTableInfo::TPtr alterData = ParseParams(path, tableInfo, storeInfo, alter, *path.DomainInfo(), status, errStr, context); if (!alterData) { result->SetError(status, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp index 42c0ad908b3..f7a47459b97 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp @@ -12,8 +12,8 @@ using namespace NKikimr; using namespace NSchemeShard; class TAlterPQ: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -27,8 +27,8 @@ class TAlterPQ: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -44,7 +44,7 @@ class TAlterPQ: public TSubOperation { return THolder(new NPQState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NPQState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -61,11 +61,11 @@ class TAlterPQ: public TSubOperation { } public: - TAlterPQ(TOperationId id, const TTxTransaction& tx) + TAlterPQ(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterPQ(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -81,14 +81,14 @@ public: TString& errStr) { TPersQueueGroupInfo::TPtr params = new TPersQueueGroupInfo(); - const bool hasKeySchema = tabletConfig->PartitionKeySchemaSize(); - + const bool hasKeySchema = tabletConfig->PartitionKeySchemaSize(); + if (alter.HasTotalGroupCount()) { - if (hasKeySchema) { - errStr = "Cannot change partition count. Use split/merge instead"; - return nullptr; - } - + if (hasKeySchema) { + errStr = "Cannot change partition count. Use split/merge instead"; + return nullptr; + } + ui32 totalGroupCount = alter.GetTotalGroupCount(); if (!totalGroupCount) { errStr = Sprintf("Invalid total groups count specified: %u", totalGroupCount); @@ -106,13 +106,13 @@ public: } if (alter.HasPQTabletConfig()) { NKikimrPQ::TPQTabletConfig alterConfig = alter.GetPQTabletConfig(); - alterConfig.ClearPartitionIds(); + alterConfig.ClearPartitionIds(); alterConfig.ClearPartitions(); - + if (!CheckPersQueueConfig(alterConfig, false, &errStr)) { return nullptr; } - + if (alterConfig.GetPartitionConfig().ExplicitChannelProfilesSize() > 0) { // Validate explicit channel profiles alter attempt const auto& ecps = alterConfig.GetPartitionConfig().GetExplicitChannelProfiles(); @@ -133,12 +133,12 @@ public: alterConfig.MutablePartitionConfig()->MutableExplicitChannelProfiles()->Swap( tabletConfig->MutablePartitionConfig()->MutableExplicitChannelProfiles()); } - - if (alterConfig.PartitionKeySchemaSize()) { - errStr = "Cannot change key schema"; - return nullptr; - } - + + if (alterConfig.PartitionKeySchemaSize()) { + errStr = "Cannot change key schema"; + return nullptr; + } + const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId()); if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) { auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id"); @@ -155,7 +155,7 @@ public: const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString(); tabletConfig->SetYdbDatabasePath(databasePath); - alterConfig.MutablePartitionKeySchema()->Swap(tabletConfig->MutablePartitionKeySchema()); + alterConfig.MutablePartitionKeySchema()->Swap(tabletConfig->MutablePartitionKeySchema()); Y_PROTOBUF_SUPPRESS_NODISCARD alterConfig.SerializeToString(¶ms->TabletConfig); alterConfig.Swap(tabletConfig); } @@ -164,16 +164,16 @@ public: return nullptr; } if (alter.PartitionsToAddSize()) { - if (hasKeySchema) { - errStr = "Cannot change partition count. Use split/merge instead"; - return nullptr; - } - + if (hasKeySchema) { + errStr = "Cannot change partition count. Use split/merge instead"; + return nullptr; + } + if (params->TotalGroupCount) { errStr = Sprintf("providing TotalGroupCount and PartitionsToAdd at the same time is forbidden"); return nullptr; } - + THashSet<ui32> parts; for (const auto& p : alter.GetPartitionsToAdd()) { if (!parts.insert(p.GetPartitionId()).second) { @@ -181,13 +181,13 @@ public: << "providing partition " << p.GetPartitionId() << " serveral times in PartitionsToAdd is forbidden"; return nullptr; } - params->PartitionsToAdd.emplace(p.GetPartitionId(), p.GetGroupId()); + params->PartitionsToAdd.emplace(p.GetPartitionId(), p.GetGroupId()); } } - if (alter.HasBootstrapConfig()) { - errStr = "Bootstrap config can be passed only upon creation"; - return nullptr; - } + if (alter.HasBootstrapConfig()) { + errStr = "Bootstrap config can be passed only upon creation"; + return nullptr; + } return params; } @@ -217,7 +217,7 @@ public: for (auto& shard : pqGroup->Shards) { auto shardIdx = shard.first; for (const auto& pqInfo : shard.second->PQInfos) { - context.SS->PersistPersQueue(db, item->PathId, shardIdx, pqInfo); + context.SS->PersistPersQueue(db, item->PathId, shardIdx, pqInfo); } } @@ -370,9 +370,9 @@ public: for (const auto& p : pqGroup->AlterData->PartitionsToAdd) { TPQShardInfo::TPersQueueInfo pqInfo; - pqInfo.PqId = p.PartitionId; - pqInfo.GroupId = p.GroupId; - pqInfo.KeyRange = p.KeyRange; + pqInfo.PqId = p.PartitionId; + pqInfo.GroupId = p.GroupId; + pqInfo.KeyRange = p.KeyRange; pqInfo.AlterVersion = alterVersion; while (it->second->PQInfos.size() >= average) { ++it; @@ -384,9 +384,9 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& alter = Transaction.GetAlterPersQueueGroup(); + const auto& alter = Transaction.GetAlterPersQueueGroup(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = alter.GetName(); const TPathId pathId = alter.HasPathId() ? context.SS->MakeLocalId(alter.GetPathId()) : InvalidPathId; @@ -414,13 +414,13 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() - .IsPQGroup() + .IsPQGroup() .NotUnderOperation() .IsCommonSensePath(); @@ -479,7 +479,7 @@ public: ui32 diff = alterData->TotalGroupCount - pqGroup->TotalGroupCount; for (ui32 i = 0; i < diff; ++i) { - alterData->PartitionsToAdd.emplace(pqGroup->NextPartitionId + i, pqGroup->TotalGroupCount + 1 + i); + alterData->PartitionsToAdd.emplace(pqGroup->NextPartitionId + i, pqGroup->TotalGroupCount + 1 + i); } if (diff > 0) { @@ -489,15 +489,15 @@ public: alterData->TotalPartitionCount = pqGroup->TotalPartitionCount + alterData->PartitionsToAdd.size(); alterData->NextPartitionId = pqGroup->NextPartitionId; for (const auto& p : alterData->PartitionsToAdd) { - if (p.GroupId == 0 || p.GroupId > alterData->TotalGroupCount) { + if (p.GroupId == 0 || p.GroupId > alterData->TotalGroupCount) { errStr = TStringBuilder() - << "Invalid partition group id " << p.GroupId + << "Invalid partition group id " << p.GroupId << " vs " << pqGroup->TotalGroupCount; result->SetError(NKikimrScheme::StatusInvalidParameter, errStr); return result; } - alterData->NextPartitionId = Max<ui32>(alterData->NextPartitionId, p.PartitionId + 1); + alterData->NextPartitionId = Max<ui32>(alterData->NextPartitionId, p.PartitionId + 1); } if (alterData->MaxPartsPerTablet < pqGroup->MaxPartsPerTablet) { @@ -561,7 +561,7 @@ public: } } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -632,16 +632,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterPQ"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterPQ"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterPQ AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -652,12 +652,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, const TTxTransaction& tx) { - return new TAlterPQ(id, tx); -} - +ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, const TTxTransaction& tx) { + return new TAlterPQ(id, tx); +} + ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterPQ(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_solomon.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_solomon.cpp index 1d02e7a4c06..1c17ee60955 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_solomon.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_solomon.cpp @@ -48,13 +48,13 @@ public: Y_VERIFY_S(solomonPartition, "rtmr partitions is null shard idx: " << shard.Idx << " Path: " << txState->TargetPathId); auto tabletId = context.SS->ShardInfos[shard.Idx].TabletID; - - if (solomonPartition->TabletId != InvalidTabletId && tabletId != solomonPartition->TabletId) { - Y_FAIL_S("Solomon partition tablet id mismatch" - << ": expected: " << solomonPartition->TabletId - << ", got: " << tabletId); - } - + + if (solomonPartition->TabletId != InvalidTabletId && tabletId != solomonPartition->TabletId) { + Y_FAIL_S("Solomon partition tablet id mismatch" + << ": expected: " << solomonPartition->TabletId + << ", got: " << tabletId); + } + solomonPartition->TabletId = tabletId; } @@ -254,27 +254,27 @@ public: TSolomonVolumeInfo::TPtr solomon = context.SS->SolomonVolumes.at(path.Base()->PathId); - if (!alter.HasPartitionCount() && !alter.GetUpdateChannelsBinding()) { + if (!alter.HasPartitionCount() && !alter.GetUpdateChannelsBinding()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "Empty alter"); return result; } - if (alter.GetUpdateChannelsBinding() && !AppData()->FeatureFlags.GetAllowUpdateChannelsBindingOfSolomonPartitions()) { + if (alter.GetUpdateChannelsBinding() && !AppData()->FeatureFlags.GetAllowUpdateChannelsBindingOfSolomonPartitions()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "Updating of channels binding is not available"); } - if (alter.HasPartitionCount()) { - if (alter.GetPartitionCount() < solomon->Partitions.size()) { + if (alter.HasPartitionCount()) { + if (alter.GetPartitionCount() < solomon->Partitions.size()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "solomon volume has more shards than requested"); - return result; - } - - if (alter.GetPartitionCount() == solomon->Partitions.size()) { + return result; + } + + if (alter.GetPartitionCount() == solomon->Partitions.size()) { result->SetError(NKikimrScheme::StatusSuccess, "solomon volume has olready the same shards as requested"); - return result; - } - } - + return result; + } + } + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; @@ -307,42 +307,42 @@ public: context.SS->PersistLastTxId(db, path.Base()); - if (alter.GetUpdateChannelsBinding()) { - txState.Shards.reserve(alter.HasPartitionCount() ? alter.GetPartitionCount() : solomon->Partitions.size()); - } else { - Y_VERIFY(alter.HasPartitionCount()); - txState.Shards.reserve(alter.GetPartitionCount() - solomon->Partitions.size()); - } + if (alter.GetUpdateChannelsBinding()) { + txState.Shards.reserve(alter.HasPartitionCount() ? alter.GetPartitionCount() : solomon->Partitions.size()); + } else { + Y_VERIFY(alter.HasPartitionCount()); + txState.Shards.reserve(alter.GetPartitionCount() - solomon->Partitions.size()); + } - if (alter.GetUpdateChannelsBinding()) { - for (const auto& [shardIdx, partitionInfo] : solomon->Partitions) { - txState.Shards.emplace_back(shardIdx, TTabletTypes::KeyValue, TTxState::CreateParts); + if (alter.GetUpdateChannelsBinding()) { + for (const auto& [shardIdx, partitionInfo] : solomon->Partitions) { + txState.Shards.emplace_back(shardIdx, TTabletTypes::KeyValue, TTxState::CreateParts); auto& shardInfo = context.SS->ShardInfos.at(shardIdx); shardInfo.CurrentTxId = OperationId.GetTxId(); shardInfo.BindedChannels = channelsBinding; - context.SS->PersistShardMapping(db, shardIdx, partitionInfo->TabletId, path.Base()->PathId, OperationId.GetTxId(), solomonPartitionInfo.TabletType); - context.SS->PersistChannelsBinding(db, shardIdx, channelsBinding); - } + context.SS->PersistShardMapping(db, shardIdx, partitionInfo->TabletId, path.Base()->PathId, OperationId.GetTxId(), solomonPartitionInfo.TabletType); + context.SS->PersistChannelsBinding(db, shardIdx, channelsBinding); + } } - if (alter.HasPartitionCount()) { - const ui64 shardsToCreate = alter.GetPartitionCount() - solomon->Partitions.size(); - - for (ui64 i = 0; i < shardsToCreate; ++i) { + if (alter.HasPartitionCount()) { + const ui64 shardsToCreate = alter.GetPartitionCount() - solomon->Partitions.size(); + + for (ui64 i = 0; i < shardsToCreate; ++i) { const auto shardIdx = context.SS->RegisterShardInfo(solomonPartitionInfo); context.SS->PersistShardMapping(db, shardIdx, InvalidTabletId, path.Base()->PathId, OperationId.GetTxId(), solomonPartitionInfo.TabletType); context.SS->PersistChannelsBinding(db, shardIdx, channelsBinding); - alterSolomon->Partitions[shardIdx] = new TSolomonPartitionInfo(solomon->Partitions.size() + i); - txState.Shards.emplace_back(shardIdx, TTabletTypes::KeyValue, TTxState::CreateParts); - } + alterSolomon->Partitions[shardIdx] = new TSolomonPartitionInfo(solomon->Partitions.size() + i); + txState.Shards.emplace_back(shardIdx, TTabletTypes::KeyValue, TTxState::CreateParts); + } context.SS->PersistUpdateNextShardIdx(db); - - path.Base()->IncShardsInside(shardsToCreate); - } - + + path.Base()->IncShardsInside(shardsToCreate); + } + solomon->AlterData = alterSolomon; context.SS->PersistAlterSolomonVolume(db, path.Base()->PathId, solomon); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_subdomain.cpp index a261c7ee991..dc1208bbb9c 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_subdomain.cpp @@ -37,8 +37,8 @@ void PersistShards(NIceDb::TNiceDb& db, TTxState& txState, ui64 shardsToCreate, } class TAlterSubDomain: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -52,8 +52,8 @@ class TAlterSubDomain: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -69,7 +69,7 @@ class TAlterSubDomain: public TSubOperation { return THolder(new NSubDomainState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NSubDomainState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -86,11 +86,11 @@ class TAlterSubDomain: public TSubOperation { } public: - TAlterSubDomain(TOperationId id, const TTxTransaction& tx) + TAlterSubDomain(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterSubDomain(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -102,10 +102,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& settings = Transaction.GetSubDomain(); + const auto& settings = Transaction.GetSubDomain(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = settings.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = settings.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterSubDomain Propose" @@ -133,8 +133,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -307,8 +307,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -356,16 +356,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterSubDomain"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterSubDomain"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterSubDomain AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -376,12 +376,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, const TTxTransaction& tx) { - return new TAlterSubDomain(id, tx); -} - +ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, const TTxTransaction& tx) { + return new TAlterSubDomain(id, tx); +} + ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterSubDomain(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp index 5192d66e8fa..b6c0b6c8b25 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp @@ -53,12 +53,12 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, status = NKikimrScheme::StatusInvalidParameter; return nullptr; } - - if (alter.HasTTLSettings()) { - errStr = "TTL on index table is not supported"; + + if (alter.HasTTLSettings()) { + errStr = "TTL on index table is not supported"; status = NKikimrScheme::StatusInvalidParameter; - return nullptr; - } + return nullptr; + } } auto copyAlter = alter; @@ -68,12 +68,12 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, copyAlter.DropColumnsSize() != 0); if (copyAlter.HasIsBackup() && copyAlter.GetIsBackup() != table->IsBackup) { - errStr = Sprintf("Cannot add/remove 'IsBackup' property"); + errStr = Sprintf("Cannot add/remove 'IsBackup' property"); status = NKikimrScheme::StatusInvalidParameter; - return nullptr; - } - - if (!hasSchemaChanges && !copyAlter.HasPartitionConfig() && !copyAlter.HasTTLSettings()) { + return nullptr; + } + + if (!hasSchemaChanges && !copyAlter.HasPartitionConfig() && !copyAlter.HasTTLSettings()) { errStr = Sprintf("No changes specified"); status = NKikimrScheme::StatusInvalidParameter; return nullptr; @@ -115,9 +115,9 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, } copyAlter.MutablePartitionConfig()->CopyFrom(compilationPartitionConfig); - const TSubDomainInfo& subDomain = *path.DomainInfo(); - const TSchemeLimits& limits = subDomain.GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(table, copyAlter, *appData->TypeRegistry, limits, subDomain, errStr); + const TSubDomainInfo& subDomain = *path.DomainInfo(); + const TSchemeLimits& limits = subDomain.GetSchemeLimits(); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(table, copyAlter, *appData->TypeRegistry, limits, subDomain, errStr); if (!alterData) { status = NKikimrScheme::StatusInvalidParameter; return nullptr; @@ -186,23 +186,23 @@ bool CheckDropingColumns(const TSchemeShard* ss, const NKikimrSchemeOp::TTableDe } } - for (const auto& child : tablePath.Base()->GetChildren()) { - const auto& childName = child.first; - const auto& childPathId = child.second; + for (const auto& child : tablePath.Base()->GetChildren()) { + const auto& childName = child.first; + const auto& childPathId = child.second; - auto childPath = ss->PathsById.at(childPathId); - if (!childPath->IsTableIndex() || childPath->Dropped()) { + auto childPath = ss->PathsById.at(childPathId); + if (!childPath->IsTableIndex() || childPath->Dropped()) { continue; } - const TTableIndexInfo::TPtr indexInfo = ss->Indexes.at(childPathId); + const TTableIndexInfo::TPtr indexInfo = ss->Indexes.at(childPathId); for (const auto& indexKey: indexInfo->IndexKeys) { if (deletedColumns.contains(indexKey)) { errStr = TStringBuilder () << "Imposible drop column because table has an index with that column" - << ", column name: " << indexKey + << ", column name: " << indexKey << ", table name: " << tablePath.PathString() - << ", index name: " << childName; + << ", index name: " << childName; return false; } } @@ -211,9 +211,9 @@ bool CheckDropingColumns(const TSchemeShard* ss, const NKikimrSchemeOp::TTableDe if (deletedColumns.contains(col)) { errStr = TStringBuilder () << "Imposible drop column because table index covers that column" - << ", column name: " << col + << ", column name: " << col << ", table name: " << tablePath.PathString() - << ", index name: " << childName; + << ", index name: " << childName; return false; } } @@ -341,28 +341,28 @@ public: TTableInfo::TPtr table = context.SS->Tables.at(pathId); table->FinishAlter(); - auto ttlIt = context.SS->TTLEnabledTables.find(pathId); - if (table->IsTTLEnabled() && ttlIt == context.SS->TTLEnabledTables.end()) { - context.SS->TTLEnabledTables[pathId] = table; - context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); - } else if (!table->IsTTLEnabled() && ttlIt != context.SS->TTLEnabledTables.end()) { - context.SS->TTLEnabledTables.erase(ttlIt); + auto ttlIt = context.SS->TTLEnabledTables.find(pathId); + if (table->IsTTLEnabled() && ttlIt == context.SS->TTLEnabledTables.end()) { + context.SS->TTLEnabledTables[pathId] = table; + context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); + } else if (!table->IsTTLEnabled() && ttlIt != context.SS->TTLEnabledTables.end()) { + context.SS->TTLEnabledTables.erase(ttlIt); context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Sub(1); - - for (ui32 i = 0; i < table->GetPartitions().size(); ++i) { - auto& shardInfo = table->GetPartitions().at(i); - if (auto& lag = shardInfo.LastCondEraseLag) { - context.SS->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); - lag.Clear(); - } - } - } - + + for (ui32 i = 0; i < table->GetPartitions().size(); ++i) { + auto& shardInfo = table->GetPartitions().at(i); + if (auto& lag = shardInfo.LastCondEraseLag) { + context.SS->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); + lag.Clear(); + } + } + } + context.SS->PersistTableAltered(db, pathId, table); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedWaitParts); return true; } @@ -392,8 +392,8 @@ public: class TAlterTable: public TSubOperation { private: - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; bool AllowShadowData = false; @@ -447,11 +447,11 @@ private: } public: - TAlterTable(TOperationId id, const TTxTransaction& tx) + TAlterTable(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TAlterTable(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -472,15 +472,15 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& alter = Transaction.GetAlterTable(); + const auto& alter = Transaction.GetAlterTable(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = alter.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = alter.GetName(); TPathId pathId; if (alter.HasId_Deprecated() || alter.HasPathId()) { pathId = alter.HasPathId() - ? PathIdFromPathId(alter.GetPathId()) + ? PathIdFromPathId(alter.GetPathId()) : context.SS->MakeLocalId(alter.GetId_Deprecated()); } @@ -503,8 +503,8 @@ public: : TPath::Resolve(parentPathStr, context.SS).Dive(name); { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -551,25 +551,25 @@ public: return result; } - if (path.Base()->GetAliveChildren() && alter.HasTTLSettings()) { - for (const auto& [_, childPathId] : path.Base()->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsTableIndex() || childPath->Dropped()) { - continue; - } - - Y_VERIFY(context.SS->Indexes.contains(childPathId)); - auto indexInfo = context.SS->Indexes.at(childPathId); - - if (indexInfo->Type == NKikimrSchemeOp::EIndexTypeGlobalAsync && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { + if (path.Base()->GetAliveChildren() && alter.HasTTLSettings()) { + for (const auto& [_, childPathId] : path.Base()->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsTableIndex() || childPath->Dropped()) { + continue; + } + + Y_VERIFY(context.SS->Indexes.contains(childPathId)); + auto indexInfo = context.SS->Indexes.at(childPathId); + + if (indexInfo->Type == NKikimrSchemeOp::EIndexTypeGlobalAsync && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "TTL is not currently supported on tables with async indices"); - return result; - } - } - } - + return result; + } + } + } + NKikimrScheme::EStatus status; TTableInfo::TAlterDataPtr alterData = ParseParams(path, table, alter, IsShadowDataAllowed(), errStr, status, context); if (!alterData) { @@ -601,16 +601,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterTable"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterTable"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterTable AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -621,12 +621,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterTable(TOperationId id, const TTxTransaction& tx) { - return new TAlterTable(id, tx); -} - +ISubOperationBase::TPtr CreateAlterTable(TOperationId id, const TTxTransaction& tx) { + return new TAlterTable(id, tx); +} + ISubOperationBase::TPtr CreateAlterTable(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TAlterTable(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_user_attrs.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_user_attrs.cpp index 16a2a0a650b..4f7dd3314e3 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_user_attrs.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_user_attrs.cpp @@ -8,28 +8,28 @@ using namespace NKikimr; using namespace NSchemeShard; class TAlterUserAttrs: public ISubOperationBase { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; public: - TAlterUserAttrs(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TAlterUserAttrs(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TAlterUserAttrs(TOperationId id) : OperationId(id) - { - } + { + } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& userAttrsPatch = Transaction.GetAlterUserAttributes(); + const auto& userAttrsPatch = Transaction.GetAlterUserAttributes(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = userAttrsPatch.GetPathName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = userAttrsPatch.GetPathName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterUserAttrs Propose" @@ -39,7 +39,7 @@ public: auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); - if (!Transaction.HasAlterUserAttributes()) { + if (!Transaction.HasAlterUserAttributes()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "UserAttributes are not present"); return result; } @@ -69,8 +69,8 @@ public: } } - TString errStr; - + TString errStr; + TUserAttributes::TPtr alterData = path.Base()->UserAttrs->CreateNextVersion(); if (!alterData->ApplyPatch(EUserAttributesOp::AlterUserAttrs, userAttrsPatch, errStr) || !alterData->CheckLimits(errStr)) @@ -79,7 +79,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -101,10 +101,10 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAlterUserAttrs"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAlterUserAttrs"); + } + void ProgressState(TOperationContext& context) override { LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterUserAttrs ProgressState" @@ -125,7 +125,7 @@ public: "TAlterUserAttrs HandleReply TEvOperationPlan" << ", opId: " << OperationId << ", stepId:" << step - << ", at schemeshard: " << ssId); + << ", at schemeshard: " << ssId); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -133,8 +133,8 @@ public: if (txState->State != TTxState::Propose) { LOG_WARN_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Duplicate PlanStep opId#" << OperationId - << " at schemeshard: " << ssId - << " txState is in state#" << TTxState::StateName(txState->State)); + << " at schemeshard: " << ssId + << " txState is in state#" << TTxState::StateName(txState->State)); return; } @@ -152,10 +152,10 @@ public: context.SS->ApplyAndPersistUserAttrs(db, path->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); context.OnComplete.UpdateTenants({pathId}); - + context.OnComplete.DoneOperation(OperationId); } @@ -164,7 +164,7 @@ public: "TAlterUserAttrs AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -175,10 +175,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, const TTxTransaction& tx) { - return new TAlterUserAttrs(id, tx); -} - +ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, const TTxTransaction& tx) { + return new TAlterUserAttrs(id, tx); +} + ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state == TTxState::Invalid || state == TTxState::Propose); return new TAlterUserAttrs(id); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp index dd61fb3bebb..c1ac9556511 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp @@ -42,7 +42,7 @@ TVector<ISubOperationBase::TPtr> ApplyBuildIndex(TOperationId nextId, const TTxT op->SetSnapshotTxId(config.GetSnaphotTxId()); op->SetBuildIndexId(config.GetBuildIndexId()); - result.push_back(CreateFinalizeBuildIndexMainTable(NextPartId(nextId, result), finalize)); + result.push_back(CreateFinalizeBuildIndexMainTable(NextPartId(nextId, result), finalize)); } { @@ -52,7 +52,7 @@ TVector<ISubOperationBase::TPtr> ApplyBuildIndex(TOperationId nextId, const TTxT alterIndex->SetName(index.LeafName()); alterIndex->SetState(NKikimrSchemeOp::EIndexState::EIndexStateReady); - result.push_back(CreateAlterTableIndex(NextPartId(nextId, result), tableIndexAltering)); + result.push_back(CreateAlterTableIndex(NextPartId(nextId, result), tableIndexAltering)); } { @@ -63,7 +63,7 @@ TVector<ISubOperationBase::TPtr> ApplyBuildIndex(TOperationId nextId, const TTxT alterTable->MutablePartitionConfig()->MutableCompactionPolicy()->SetKeepEraseMarkers(false); alterTable->MutablePartitionConfig()->SetShadowData(false); - result.push_back(CreateFinalizeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableAltering)); + result.push_back(CreateFinalizeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableAltering)); } return result; @@ -93,15 +93,15 @@ TVector<ISubOperationBase::TPtr> CancelBuildIndex(TOperationId nextId, const TTx op->SetSnapshotTxId(config.GetSnaphotTxId()); op->SetBuildIndexId(config.GetBuildIndexId()); - result.push_back(CreateFinalizeBuildIndexMainTable(NextPartId(nextId, result), finalize)); + result.push_back(CreateFinalizeBuildIndexMainTable(NextPartId(nextId, result), finalize)); } { auto tableIndexDropping = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); auto operation = tableIndexDropping.MutableDrop(); operation->SetName(ToString(index.Base()->Name)); - - result.push_back(CreateDropTableIndex(NextPartId(nextId, result), tableIndexDropping)); + + result.push_back(CreateDropTableIndex(NextPartId(nextId, result), tableIndexDropping)); } Y_VERIFY(index.Base()->GetChildren().size() == 1); @@ -133,8 +133,8 @@ TVector<ISubOperationBase::TPtr> CancelBuildIndex(TOperationId nextId, const TTx auto implTableDropping = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); auto operation = implTableDropping.MutableDrop(); operation->SetName(ToString(implTable.Base()->Name)); - - result.push_back(CreateDropTable(NextPartId(nextId,result), implTableDropping)); + + result.push_back(CreateDropTable(NextPartId(nextId,result), implTableDropping)); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_assign_bsv.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_assign_bsv.cpp index 58b264d12c0..de1576f2b18 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_assign_bsv.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_assign_bsv.cpp @@ -7,32 +7,32 @@ using namespace NKikimr; using namespace NSchemeShard; class TAssignBlockStoreVolume: public ISubOperationBase { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; public: - TAssignBlockStoreVolume(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TAssignBlockStoreVolume(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TAssignBlockStoreVolume(TOperationId id) : OperationId(id) - { - } + { + } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetAssignBlockStoreVolume().GetName(); - const TString mountToken = Transaction.GetAssignBlockStoreVolume().GetNewMountToken(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetAssignBlockStoreVolume().GetName(); + const TString mountToken = Transaction.GetAssignBlockStoreVolume().GetNewMountToken(); const auto version = Transaction.GetAssignBlockStoreVolume().GetTokenVersion(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAssignBlockStoreVolume Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", operationId: " << OperationId << ", at schemeshard: " << ssId); @@ -42,8 +42,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -81,8 +81,8 @@ public: return result; } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -93,16 +93,16 @@ public: ++volume->TokenVersion; context.SS->PersistBlockStoreVolumeMountToken(db, path.Base()->PathId, volume); - context.OnComplete.PublishToSchemeBoard(OperationId, path.Base()->PathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, path.Base()->PathId); + context.OnComplete.DoneOperation(OperationId); return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TAssignBlockStoreVolume"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TAssignBlockStoreVolume"); + } + void ProgressState(TOperationContext&) override { Y_FAIL("no progress state for assign bsc"); return; @@ -118,10 +118,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, const TTxTransaction& tx) { - return new TAssignBlockStoreVolume(id, tx); -} - +ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, const TTxTransaction& tx) { + return new TAssignBlockStoreVolume(id, tx); +} + ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state == TTxState::Invalid || state == TTxState::Propose); return new TAssignBlockStoreVolume(id); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h index 599a6802edb..640f5f13565 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h @@ -1,32 +1,32 @@ #pragma once #include "schemeshard__operation_part.h" #include "schemeshard__operation_common.h" -#include "schemeshard_billing_helpers.h" +#include "schemeshard_billing_helpers.h" #include "schemeshard_impl.h" #include <ydb/core/base/subdomain.h> #include <ydb/core/metering/metering.h> -#include <util/generic/utility.h> - -namespace NKikimr { +#include <util/generic/utility.h> + +namespace NKikimr { namespace NSchemeShard { -template <typename TKind> +template <typename TKind> class TConfigurePart: public TSubOperationState { - const TTxState::ETxType TxType; - const TOperationId OperationId; + const TTxState::ETxType TxType; + const TOperationId OperationId; TString DebugHint() const override { return TStringBuilder() - << TKind::Name() << " TConfigurePart" - << ", opId: " << OperationId; + << TKind::Name() << " TConfigurePart" + << ", opId: " << OperationId; } public: - TConfigurePart(TTxState::ETxType type, TOperationId id) - : TxType(type) - , OperationId(id) + TConfigurePart(TTxState::ETxType type, TOperationId id) + : TxType(type) + , OperationId(id) { IgnoreMessages(DebugHint(), {}); } @@ -46,40 +46,40 @@ public: auto ssId = context.SS->SelfTabletId(); LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - TKind::Name() << " TConfigurePart ProgressState" - << ", opId: " << OperationId - << ", at schemeshard: " << ssId); + TKind::Name() << " TConfigurePart ProgressState" + << ", opId: " << OperationId + << ", at schemeshard: " << ssId); TTxState* txState = context.SS->TxInFlight.FindPtr(OperationId); Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TxType); + Y_VERIFY(txState->TxType == TxType); Y_VERIFY(txState->State == TTxState::ConfigureParts); txState->ClearShardsInProgress(); - TKind::ProposeTx(OperationId, *txState, context); - txState->UpdateShardsInProgress(TTxState::ConfigureParts); + TKind::ProposeTx(OperationId, *txState, context); + txState->UpdateShardsInProgress(TTxState::ConfigureParts); return false; } }; -template <typename TKind> +template <typename TKind> class TProposedWaitParts: public TSubOperationState { -protected: - const TTxState::ETxType TxType; - const TOperationId OperationId; - +protected: + const TTxState::ETxType TxType; + const TOperationId OperationId; + private: TString DebugHint() const override { return TStringBuilder() - << TKind::Name() << " TProposedWaitParts" - << ", opId: " << OperationId; + << TKind::Name() << " TProposedWaitParts" + << ", opId: " << OperationId; } public: - TProposedWaitParts(TTxState::ETxType type, TOperationId id) - : TxType(type) - , OperationId(id) + TProposedWaitParts(TTxState::ETxType type, TOperationId id) + : TxType(type) + , OperationId(id) { IgnoreMessages(DebugHint(), { TEvHive::TEvCreateTabletReply::EventType @@ -88,69 +88,69 @@ public: ); } - static void Bill(TOperationId operationId, const TPathId& pathId, const TShardIdx& shardIdx, ui64 ru, TOperationContext& context) { - const auto path = TPath::Init(pathId, context.SS); - const auto domainId = path.DomainId(); - const auto domain = TPath::Init(domainId, context.SS); - - auto unableToMakeABill = [&](const TStringBuf reason) { - LOG_WARN_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to make a bill" - << ": kind# " << TKind::Name() - << ", opId# " << operationId - << ", reason# " << reason - << ", domain# " << domain.PathString() - << ", domainId# " << domainId - << ", IsDomainSchemeShard: " << context.SS->IsDomainSchemeShard - << ", ParentDomainId: " << context.SS->ParentDomainId - << ", ResourcesDomainId: " << domain.DomainInfo()->GetResourcesDomainId()); - }; - - if (!context.SS->IsServerlessDomain(domain)) { - return unableToMakeABill("domain is not a serverless db"); - } - - const auto& attrs = domain.Base()->UserAttrs->Attrs; - if (!attrs.contains("cloud_id")) { - return unableToMakeABill("cloud_id not found in user attributes"); - } - - if (!attrs.contains("folder_id")) { - return unableToMakeABill("folder_id not found in user attributes"); - } - - if (!attrs.contains("database_id")) { - return unableToMakeABill("database_id not found in user attributes"); - } - - if (!TKind::NeedToBill(pathId, context)) { - return unableToMakeABill("does not need to bill"); - } - - const auto now = context.Ctx.Now(); - const TString id = TStringBuilder() << operationId.GetTxId() - << "-" << pathId.OwnerId << "-" << pathId.LocalPathId - << "-" << shardIdx.GetOwnerId() << "-" << shardIdx.GetLocalId(); - - const TString billRecord = TBillRecord() - .Id(id) - .CloudId(attrs.at("cloud_id")) - .FolderId(attrs.at("folder_id")) - .ResourceId(attrs.at("database_id")) - .SourceWt(now) - .Usage(TBillRecord::RequestUnits(Max(ui64(1), ru), now)) - .ToString(); - - LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Make a bill" - << ": kind# " << TKind::Name() - << ", opId# " << operationId - << ", domain# " << domain.PathString() - << ", domainId# " << domainId - << ", record# " << billRecord); - - context.OnComplete.Send(NMetering::MakeMeteringServiceID(), - new NMetering::TEvMetering::TEvWriteMeteringJson(std::move(billRecord))); - } - + static void Bill(TOperationId operationId, const TPathId& pathId, const TShardIdx& shardIdx, ui64 ru, TOperationContext& context) { + const auto path = TPath::Init(pathId, context.SS); + const auto domainId = path.DomainId(); + const auto domain = TPath::Init(domainId, context.SS); + + auto unableToMakeABill = [&](const TStringBuf reason) { + LOG_WARN_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to make a bill" + << ": kind# " << TKind::Name() + << ", opId# " << operationId + << ", reason# " << reason + << ", domain# " << domain.PathString() + << ", domainId# " << domainId + << ", IsDomainSchemeShard: " << context.SS->IsDomainSchemeShard + << ", ParentDomainId: " << context.SS->ParentDomainId + << ", ResourcesDomainId: " << domain.DomainInfo()->GetResourcesDomainId()); + }; + + if (!context.SS->IsServerlessDomain(domain)) { + return unableToMakeABill("domain is not a serverless db"); + } + + const auto& attrs = domain.Base()->UserAttrs->Attrs; + if (!attrs.contains("cloud_id")) { + return unableToMakeABill("cloud_id not found in user attributes"); + } + + if (!attrs.contains("folder_id")) { + return unableToMakeABill("folder_id not found in user attributes"); + } + + if (!attrs.contains("database_id")) { + return unableToMakeABill("database_id not found in user attributes"); + } + + if (!TKind::NeedToBill(pathId, context)) { + return unableToMakeABill("does not need to bill"); + } + + const auto now = context.Ctx.Now(); + const TString id = TStringBuilder() << operationId.GetTxId() + << "-" << pathId.OwnerId << "-" << pathId.LocalPathId + << "-" << shardIdx.GetOwnerId() << "-" << shardIdx.GetLocalId(); + + const TString billRecord = TBillRecord() + .Id(id) + .CloudId(attrs.at("cloud_id")) + .FolderId(attrs.at("folder_id")) + .ResourceId(attrs.at("database_id")) + .SourceWt(now) + .Usage(TBillRecord::RequestUnits(Max(ui64(1), ru), now)) + .ToString(); + + LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Make a bill" + << ": kind# " << TKind::Name() + << ", opId# " << operationId + << ", domain# " << domain.PathString() + << ", domainId# " << domainId + << ", record# " << billRecord); + + context.OnComplete.Send(NMetering::MakeMeteringServiceID(), + new NMetering::TEvMetering::TEvWriteMeteringJson(std::move(billRecord))); + } + static void CollectStats(TOperationId operationId, const TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) { const auto& evRecord = ev->Get()->Record; @@ -175,31 +175,31 @@ public: } NIceDb::TNiceDb db(context.Txc.DB); - const auto& result = evRecord.GetOpResult(); - - if (!txState.ShardStatuses.contains(shardIdx)) { - auto& shardStatus = txState.ShardStatuses[shardIdx]; - shardStatus.Success = result.GetSuccess(); - shardStatus.Error = result.GetExplain(); - shardStatus.BytesProcessed = result.GetBytesProcessed(); - shardStatus.RowsProcessed = result.GetRowsProcessed(); - context.SS->PersistTxShardStatus(db, operationId, shardIdx, shardStatus); - - const ui64 ru = TKind::RequestUnits(shardStatus.BytesProcessed, shardStatus.RowsProcessed); - Bill(operationId, txState.TargetPathId, shardIdx, ru, context); - } - - // TODO(ilnaz): backward compatability, remove it - if (result.GetSuccess()) { - if (result.HasBytesProcessed()) { - txState.DataTotalSize += result.GetBytesProcessed(); + const auto& result = evRecord.GetOpResult(); + + if (!txState.ShardStatuses.contains(shardIdx)) { + auto& shardStatus = txState.ShardStatuses[shardIdx]; + shardStatus.Success = result.GetSuccess(); + shardStatus.Error = result.GetExplain(); + shardStatus.BytesProcessed = result.GetBytesProcessed(); + shardStatus.RowsProcessed = result.GetRowsProcessed(); + context.SS->PersistTxShardStatus(db, operationId, shardIdx, shardStatus); + + const ui64 ru = TKind::RequestUnits(shardStatus.BytesProcessed, shardStatus.RowsProcessed); + Bill(operationId, txState.TargetPathId, shardIdx, ru, context); + } + + // TODO(ilnaz): backward compatability, remove it + if (result.GetSuccess()) { + if (result.HasBytesProcessed()) { + txState.DataTotalSize += result.GetBytesProcessed(); db.Table<Schema::TxInFlightV2>().Key(operationId.GetTxId(), operationId.GetSubTxId()).Update( - NIceDb::TUpdate<Schema::TxInFlightV2::DataTotalSize>(txState.DataTotalSize)); + NIceDb::TUpdate<Schema::TxInFlightV2::DataTotalSize>(txState.DataTotalSize)); } } else { - if (result.HasExplain()) { - TString explain = result.GetExplain(); + if (result.HasExplain()) { + TString explain = result.GetExplain(); if (context.SS->IsLocalId(shardIdx)) { db.Table<Schema::ShardBackupStatus>().Key(operationId.GetTxId(), shardIdx.GetLocalId()).Update( @@ -217,8 +217,8 @@ public: const auto& evRecord = ev->Get()->Record; LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" - << " at tablet# " << ssId + DebugHint() << " HandleReply TEvSchemaChanged" + << " at tablet# " << ssId << " message# " << evRecord.ShortDebugString()); bool allnotificationRecieved = NTableState::CollectSchemaChanged(OperationId, ev, context); @@ -235,7 +235,7 @@ public: return false; } - TKind::FinishStats(OperationId, txState, context); + TKind::FinishStats(OperationId, txState, context); return true; } @@ -260,8 +260,8 @@ public: } txState->UpdateShardsInProgress(TTxState::ProposedWaitParts); - if (txState->Cancel) { - context.SS->ChangeTxState(db, OperationId, TTxState::Aborting); + if (txState->Cancel) { + context.SS->ChangeTxState(db, OperationId, TTxState::Aborting); return true; } @@ -275,7 +275,7 @@ public: if (txState->ShardsInProgress.empty()) { NTableState::AckAllSchemaChanges(OperationId, *txState, context); context.SS->ChangeTxState(db, OperationId, TTxState::Done); - TKind::FinishStats(OperationId, *txState, context); + TKind::FinishStats(OperationId, *txState, context); return true; } @@ -283,22 +283,22 @@ public: } }; -template <typename TKind, typename TEvCancel> -class TAborting: public TProposedWaitParts<TKind> { - using TProposedWaitParts<TKind>::OperationId; - using TProposedWaitParts<TKind>::TxType; +template <typename TKind, typename TEvCancel> +class TAborting: public TProposedWaitParts<TKind> { + using TProposedWaitParts<TKind>::OperationId; + using TProposedWaitParts<TKind>::TxType; TString DebugHint() const override { return TStringBuilder() - << TKind::Name() << " TAborting" - << ", opId: " << OperationId; + << TKind::Name() << " TAborting" + << ", opId: " << OperationId; } public: - TAborting(TTxState::ETxType type, TOperationId id) - : TProposedWaitParts<TKind>(type, id) + TAborting(TTxState::ETxType type, TOperationId id) + : TProposedWaitParts<TKind>(type, id) { - this->IgnoreMessages(DebugHint(), + this->IgnoreMessages(DebugHint(), { TEvHive::TEvCreateTabletReply::EventType , TEvDataShard::TEvProposeTransactionResult::EventType , TEvPrivate::TEvOperationPlan::EventType } @@ -314,8 +314,8 @@ public: TTxState* txState = context.SS->TxInFlight.FindPtr(OperationId); Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TxType); - Y_VERIFY(txState->State == TTxState::Aborting); + Y_VERIFY(txState->TxType == TxType); + Y_VERIFY(txState->State == TTxState::Aborting); txState->ClearShardsInProgress(); @@ -333,14 +333,14 @@ public: auto idx = txState->Shards[i].Idx; auto datashardId = context.SS->ShardInfos[idx].TabletID; - LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - TKind::Name() << " Abort" - << ", on datashard: " << datashardId - << ", opId: " << OperationId - << ", at schemeshard: " << context.SS->TabletID()); + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + TKind::Name() << " Abort" + << ", on datashard: " << datashardId + << ", opId: " << OperationId + << ", at schemeshard: " << context.SS->TabletID()); - THolder<TEvCancel> event = - THolder(new TEvCancel(ui64(OperationId.GetTxId()), txState->TargetPathId.LocalPathId)); + THolder<TEvCancel> event = + THolder(new TEvCancel(ui64(OperationId.GetTxId()), txState->TargetPathId.LocalPathId)); context.OnComplete.BindMsgToPipe(OperationId, datashardId, idx, event.Release()); } @@ -353,7 +353,7 @@ public: if (txState->ShardsInProgress.empty()) { NTableState::AckAllSchemaChanges(OperationId, *txState, context); context.SS->ChangeTxState(db, OperationId, TTxState::Done); - TKind::FinishStats(OperationId, *txState, context); + TKind::FinishStats(OperationId, *txState, context); return true; } @@ -361,21 +361,21 @@ public: } }; -template <typename TKind> +template <typename TKind> class TPropose: public TSubOperationState { - const TTxState::ETxType TxType; - const TOperationId OperationId; + const TTxState::ETxType TxType; + const TOperationId OperationId; TString DebugHint() const override { return TStringBuilder() - << TKind::Name() << " TPropose" - << ", opId: " << OperationId; + << TKind::Name() << " TPropose" + << ", opId: " << OperationId; } - + public: - TPropose(TTxState::ETxType type, TOperationId id) - : TxType(type) - , OperationId(id) + TPropose(TTxState::ETxType type, TOperationId id) + : TxType(type) + , OperationId(id) { IgnoreMessages(DebugHint(), {TEvDataShard::TEvProposeTransactionResult::EventType}); } @@ -389,7 +389,7 @@ public: << ", at schemeshard: " << ssId); NTableState::CollectSchemaChanged(OperationId, ev, context); - TProposedWaitParts<TKind>::CollectStats(OperationId, ev, context); + TProposedWaitParts<TKind>::CollectStats(OperationId, ev, context); return false; } @@ -404,19 +404,19 @@ public: TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TxType); + Y_VERIFY(txState->TxType == TxType); TPathId pathId = txState->TargetPathId; TPathElement::TPtr path = context.SS->PathsById.at(pathId); - TKind::PersistDone(pathId, context); + TKind::PersistDone(pathId, context); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - NIceDb::TNiceDb db(context.Txc.DB); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + NIceDb::TNiceDb db(context.Txc.DB); context.SS->ChangeTxState(db, OperationId, TTxState::ProposedWaitParts); - + return true; } @@ -429,7 +429,7 @@ public: TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TxType); + Y_VERIFY(txState->TxType == TxType); TSet<TTabletId> shardSet; for (const auto& shard : txState->Shards) { @@ -443,13 +443,13 @@ public: } }; -template <typename TKind, typename TEvCancel> -class TBackupRestoreOperationBase: public TSubOperation { - const TTxState::ETxType TxType; - const TPathElement::EPathState Lock; +template <typename TKind, typename TEvCancel> +class TBackupRestoreOperationBase: public TSubOperation { + const TTxState::ETxType TxType; + const TPathElement::EPathState Lock; - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -471,19 +471,19 @@ class TBackupRestoreOperationBase: public TSubOperation { case TTxState::ProposedWaitParts: { TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TxType); - - if (txState->Cancel) { - if (txState->State == TTxState::Done) { - return TTxState::Done; - } - - Y_VERIFY(txState->State == TTxState::Aborting); - return TTxState::Aborting; + Y_VERIFY(txState->TxType == TxType); + + if (txState->Cancel) { + if (txState->State == TTxState::Done) { + return TTxState::Done; + } + + Y_VERIFY(txState->State == TTxState::Aborting); + return TTxState::Aborting; } return TTxState::Done; } - case TTxState::Aborting: + case TTxState::Aborting: return TTxState::Done; default: @@ -498,13 +498,13 @@ class TBackupRestoreOperationBase: public TSubOperation { case TTxState::CreateParts: return THolder(new TCreateParts(OperationId)); case TTxState::ConfigureParts: - return THolder(new TConfigurePart<TKind>(TxType, OperationId)); + return THolder(new TConfigurePart<TKind>(TxType, OperationId)); case TTxState::Propose: - return THolder(new TPropose<TKind>(TxType, OperationId)); + return THolder(new TPropose<TKind>(TxType, OperationId)); case TTxState::ProposedWaitParts: - return THolder(new TProposedWaitParts<TKind>(TxType, OperationId)); - case TTxState::Aborting: - return THolder(new TAborting<TKind, TEvCancel>(TxType, OperationId)); + return THolder(new TProposedWaitParts<TKind>(TxType, OperationId)); + case TTxState::Aborting: + return THolder(new TAborting<TKind, TEvCancel>(TxType, OperationId)); case TTxState::Done: return THolder(new TDone(OperationId)); default: @@ -522,36 +522,36 @@ class TBackupRestoreOperationBase: public TSubOperation { } public: - TBackupRestoreOperationBase( - TTxState::ETxType type, TPathElement::EPathState lock, - TOperationId id, const TTxTransaction& tx) - : TxType(type) - , Lock(lock) - , OperationId(id) - , Transaction(tx) - { - } - - TBackupRestoreOperationBase( - TTxState::ETxType type, TPathElement::EPathState lock, - TOperationId id, TTxState::ETxState state) - : TxType(type) - , Lock(lock) - , OperationId(id) + TBackupRestoreOperationBase( + TTxState::ETxType type, TPathElement::EPathState lock, + TOperationId id, const TTxTransaction& tx) + : TxType(type) + , Lock(lock) + , OperationId(id) + , Transaction(tx) + { + } + + TBackupRestoreOperationBase( + TTxState::ETxType type, TPathElement::EPathState lock, + TOperationId id, TTxState::ETxState state) + : TxType(type) + , Lock(lock) + , OperationId(id) , State(state) { SetState(SelectStateFunc(state)); } - void PrepareChanges(TPathElement::TPtr path, TOperationContext& context) { - Y_VERIFY(context.SS->Tables.contains(path->PathId)); - TTableInfo::TPtr& table = context.SS->Tables.at(path->PathId); + void PrepareChanges(TPathElement::TPtr path, TOperationContext& context) { + Y_VERIFY(context.SS->Tables.contains(path->PathId)); + TTableInfo::TPtr& table = context.SS->Tables.at(path->PathId); + + path->LastTxId = OperationId.GetTxId(); + path->PathState = Lock; - path->LastTxId = OperationId.GetTxId(); - path->PathState = Lock; + TTxState& txState = context.SS->CreateTx(OperationId, TxType, path->PathId); - TTxState& txState = context.SS->CreateTx(OperationId, TxType, path->PathId); - txState.Shards.reserve(table->GetPartitions().size()); for (const auto& shard : table->GetPartitions()) { auto shardIdx = shard.ShardIdx; @@ -561,55 +561,55 @@ public: NIceDb::TNiceDb db(context.Txc.DB); context.SS->PersistTxState(db, OperationId); - TKind::PersistTask(path->PathId, Transaction, context); + TKind::PersistTask(path->PathId, Transaction, context); for (auto splitTx: table->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); + context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); } txState.State = TTxState::CreateParts; - context.OnComplete.ActivateTx(OperationId); + context.OnComplete.ActivateTx(OperationId); } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const TString& parentPath = Transaction.GetWorkingDir(); - const TString name = TKind::GetTableName(Transaction); + const TString& parentPath = Transaction.GetWorkingDir(); + const TString name = TKind::GetTableName(Transaction); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - TKind::Name() << " Propose" + TKind::Name() << " Propose" << ", path: " << parentPath << "/" << name << ", opId: " << OperationId << ", at schemeshard: " << ssId); auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); - if (!Transaction.HasWorkingDir()) { + if (!Transaction.HasWorkingDir()) { result->SetError(NKikimrScheme::StatusInvalidParameter, - "Malformed request: no working dir"); + "Malformed request: no working dir"); return result; } - if (!TKind::HasTask(Transaction)) { + if (!TKind::HasTask(Transaction)) { result->SetError( NKikimrScheme::StatusInvalidParameter, - "Malformed request"); + "Malformed request"); return result; } - if (name.empty()) { + if (name.empty()) { result->SetError( NKikimrScheme::StatusInvalidParameter, - "No table name in task"); + "No table name in task"); return result; } TPath path = TPath::Resolve(parentPath, context.SS).Dive(name); { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -628,8 +628,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -639,27 +639,27 @@ public: return result; } - PrepareChanges(path.Base(), context); + PrepareChanges(path.Base(), context); State = NextState(); SetState(SelectStateFunc(State)); return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TBackupRestoreOperationBase"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TBackupRestoreOperationBase"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - TKind::Name() << " AbortUnsafe" + TKind::Name() << " AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } }; } -} +} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_cancel_tx.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_cancel_tx.cpp index 76ac583836f..aceebc37860 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_cancel_tx.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_cancel_tx.cpp @@ -6,7 +6,7 @@ namespace { using namespace NKikimr; using namespace NSchemeShard; -class TTxCancelTx: public ISubOperationBase { +class TTxCancelTx: public ISubOperationBase { private: TEvSchemeShard::TEvCancelTx::TPtr Event; @@ -19,37 +19,37 @@ public: const auto& evRecord = Event->Get()->Record; Y_VERIFY(evRecord.HasTxId()); - Y_VERIFY(evRecord.HasTargetTxId()); + Y_VERIFY(evRecord.HasTargetTxId()); LOG_DEBUG(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Execute cancel tx txid #%" PRIu64, - evRecord.GetTargetTxId()); + "Execute cancel tx txid #%" PRIu64, + evRecord.GetTargetTxId()); auto proposeResult = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, evRecord.GetTxId(), context.SS->TabletID()); THolder<TEvSchemeShard::TEvCancelTxResult> result = MakeHolder<TEvSchemeShard::TEvCancelTxResult>(evRecord.GetTargetTxId(), evRecord.GetTxId()); - TOperationId targetOpId = TOperationId(evRecord.GetTargetTxId(), 0); + TOperationId targetOpId = TOperationId(evRecord.GetTargetTxId(), 0); - auto found = context.SS->FindTx(targetOpId); + auto found = context.SS->FindTx(targetOpId); if (!found) { result->Record.SetStatus(NKikimrScheme::StatusTxIdNotExists); - result->Record.SetResult("Transaction not found"); - context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); + result->Record.SetResult("Transaction not found"); + context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); return proposeResult; } TTxState& txState = *found; - if (txState.TxType != TTxState::TxBackup && txState.TxType != TTxState::TxRestore) { + if (txState.TxType != TTxState::TxBackup && txState.TxType != TTxState::TxRestore) { result->Record.SetStatus(NKikimrScheme::StatusTxIsNotCancellable); - result->Record.SetResult("Transaction is not cancellable"); - context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); + result->Record.SetResult("Transaction is not cancellable"); + context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); return proposeResult; } - if (txState.State == TTxState::Aborting) { + if (txState.State == TTxState::Aborting) { result->Record.SetStatus(NKikimrScheme::StatusAccepted); - result->Record.SetResult("Tx is cancelling at SchemeShard already"); + result->Record.SetResult("Tx is cancelling at SchemeShard already"); context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); return proposeResult; } @@ -57,28 +57,28 @@ public: NIceDb::TNiceDb db(context.Txc.DB); /* Let's abort at the final stage of the transaction */ - txState.Cancel = true; - context.SS->PersistCancelTx(db, targetOpId, txState); + txState.Cancel = true; + context.SS->PersistCancelTx(db, targetOpId, txState); result->Record.SetStatus(NKikimrScheme::StatusAccepted); result->Record.SetResult("Cancelled at SchemeShard"); - context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); + context.OnComplete.Send(Event->Sender, std::move(result), evRecord.GetTxId()); - context.OnComplete.ActivateTx(targetOpId); + context.OnComplete.ActivateTx(targetOpId); return proposeResult; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TTxCancelTx"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TTxCancelTx"); + } + void ProgressState(TOperationContext&) override { - Y_FAIL("no progress state for cancel tx"); + Y_FAIL("no progress state for cancel tx"); return; } void AbortUnsafe(TTxId, TOperationContext&) override { - Y_FAIL("no AbortUnsafe for cancel tx"); + Y_FAIL("no AbortUnsafe for cancel tx"); } }; @@ -90,7 +90,7 @@ namespace NSchemeShard { ISubOperationBase::TPtr CreateTxCancelTx(TEvSchemeShard::TEvCancelTx::TPtr ev) { - return new TTxCancelTx(ev); + return new TTxCancelTx(ev); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp index 21dade0bf13..5a88636ffae 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp @@ -148,14 +148,14 @@ bool NTableState::CollectSchemaChanged( if (evRecord.HasOpResult()) { // TODO: remove TxBackup handling - Y_VERIFY_DEBUG(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); + Y_VERIFY_DEBUG(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); } if (!txState.ReadyForNotifications) { return false; } - if (txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore) { - Y_VERIFY(txState.State == TTxState::ProposedWaitParts || txState.State == TTxState::Aborting); + if (txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore) { + Y_VERIFY(txState.State == TTxState::ProposedWaitParts || txState.State == TTxState::Aborting); } else { Y_VERIFY(txState.State == TTxState::ProposedWaitParts); } @@ -260,20 +260,20 @@ void NTableState::UpdatePartitioningForTableModification(TOperationId operationI commonShardOp = TTxState::DropParts; } else if (txState.TxType == TTxState::TxBackup) { commonShardOp = TTxState::ConfigureParts; - } else if (txState.TxType == TTxState::TxRestore) { - commonShardOp = TTxState::ConfigureParts; + } else if (txState.TxType == TTxState::TxRestore) { + commonShardOp = TTxState::ConfigureParts; } else if (txState.TxType == TTxState::TxInitializeBuildIndex) { commonShardOp = TTxState::ConfigureParts; } else if (txState.TxType == TTxState::TxFinalizeBuildIndex) { commonShardOp = TTxState::ConfigureParts; } else if (txState.TxType == TTxState::TxDropTableIndexAtMainTable) { commonShardOp = TTxState::ConfigureParts; - } else if (txState.TxType == TTxState::TxCreateCdcStreamAtTable) { - commonShardOp = TTxState::ConfigureParts; - } else if (txState.TxType == TTxState::TxAlterCdcStreamAtTable) { - commonShardOp = TTxState::ConfigureParts; - } else if (txState.TxType == TTxState::TxDropCdcStreamAtTable) { - commonShardOp = TTxState::ConfigureParts; + } else if (txState.TxType == TTxState::TxCreateCdcStreamAtTable) { + commonShardOp = TTxState::ConfigureParts; + } else if (txState.TxType == TTxState::TxAlterCdcStreamAtTable) { + commonShardOp = TTxState::ConfigureParts; + } else if (txState.TxType == TTxState::TxDropCdcStreamAtTable) { + commonShardOp = TTxState::ConfigureParts; } else { Y_FAIL("UNREACHABLE"); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_common.h index 84d3bd3dd98..5a3da32cade 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.h @@ -333,7 +333,7 @@ public: path.Base()->Owner, path.GetEffectiveACL(), path.GetEffectiveACLVersion(), processing, storagePools, path.Base()->UserAttrs->Attrs, path.Base()->UserAttrs->AlterVersion, - schemeLimits, ui64(alterData->GetSharedHive()), alterData->GetResourcesDomainId() + schemeLimits, ui64(alterData->GetSharedHive()), alterData->GetResourcesDomainId() ); if (alterData->GetDeclaredSchemeQuotas()) { event->Record.MutableDeclaredSchemeQuotas()->CopyFrom(*alterData->GetDeclaredSchemeQuotas()); @@ -425,14 +425,14 @@ public: ++parentDir->DirAlterVersion; context.SS->PersistPathDirAlterVersion(db, parentDir); context.SS->ClearDescribePathCaches(parentDir); - context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); + context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); } context.OnComplete.UpdateTenant(pathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -681,21 +681,21 @@ public: objectDomain->SetPathId(domainId.LocalPathId); } - Y_VERIFY(context.SS->SubDomains.contains(domainId)); - TSubDomainInfo::TPtr subDomain = context.SS->SubDomains.at(domainId); - - TPathId resourcesDomainId; - if (subDomain->GetResourcesDomainId()) { - resourcesDomainId = subDomain->GetResourcesDomainId(); - } else if (subDomain->GetAlter() && subDomain->GetAlter()->GetResourcesDomainId()) { - resourcesDomainId = subDomain->GetAlter()->GetResourcesDomainId(); - } else { - Y_FAIL("Cannot retrieve resources domain id"); - } - + Y_VERIFY(context.SS->SubDomains.contains(domainId)); + TSubDomainInfo::TPtr subDomain = context.SS->SubDomains.at(domainId); + + TPathId resourcesDomainId; + if (subDomain->GetResourcesDomainId()) { + resourcesDomainId = subDomain->GetResourcesDomainId(); + } else if (subDomain->GetAlter() && subDomain->GetAlter()->GetResourcesDomainId()) { + resourcesDomainId = subDomain->GetAlter()->GetResourcesDomainId(); + } else { + Y_FAIL("Cannot retrieve resources domain id"); + } + auto allowedDomain = ev->Record.AddAllowedDomains(); - allowedDomain->SetSchemeShard(resourcesDomainId.OwnerId); - allowedDomain->SetPathId(resourcesDomainId.LocalPathId); + allowedDomain->SetSchemeShard(resourcesDomainId.OwnerId); + allowedDomain->SetPathId(resourcesDomainId.LocalPathId); if (tablePartitionConfig) { if (tablePartitionConfig->FollowerGroupsSize()) { @@ -760,8 +760,8 @@ public: if (txState->TxType == TTxState::TxDropTable || txState->TxType == TTxState::TxAlterTable - || txState->TxType == TTxState::TxBackup - || txState->TxType == TTxState::TxRestore) { + || txState->TxType == TTxState::TxBackup + || txState->TxType == TTxState::TxRestore) { if (NTableState::CheckPartitioningChangedForTableModification(*txState, context)) { LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, DebugHint() << " ProgressState" @@ -1024,21 +1024,21 @@ public: event->Record.MutableTabletConfig()->SetVersion(pqGroup->AlterVersion + 1); for (const auto& pq : pqShard->PQInfos) { - event->Record.MutableTabletConfig()->AddPartitionIds(pq.PqId); - - auto& partition = *event->Record.MutableTabletConfig()->AddPartitions(); - partition.SetPartitionId(pq.PqId); - if (pq.KeyRange) { - pq.KeyRange->SerializeToProto(*partition.MutableKeyRange()); - } + event->Record.MutableTabletConfig()->AddPartitionIds(pq.PqId); + + auto& partition = *event->Record.MutableTabletConfig()->AddPartitions(); + partition.SetPartitionId(pq.PqId); + if (pq.KeyRange) { + pq.KeyRange->SerializeToProto(*partition.MutableKeyRange()); + } + } + + if (pqGroup->AlterData && pqGroup->AlterData->BootstrapConfig) { + Y_VERIFY(txState->TxType == TTxState::TxCreatePQGroup); + const bool ok = ParseFromStringNoSizeLimit(*event->Record.MutableBootstrapConfig(), pqGroup->AlterData->BootstrapConfig); + Y_VERIFY(ok); } - if (pqGroup->AlterData && pqGroup->AlterData->BootstrapConfig) { - Y_VERIFY(txState->TxType == TTxState::TxCreatePQGroup); - const bool ok = ParseFromStringNoSizeLimit(*event->Record.MutableBootstrapConfig(), pqGroup->AlterData->BootstrapConfig); - Y_VERIFY(ok); - } - context.OnComplete.BindMsgToPipe(OperationId, tabletId, idx, event.Release()); } else { Y_VERIFY(shard.TabletType == ETabletType::PersQueueReadBalancer); @@ -1160,7 +1160,7 @@ public: context.SS->PersistPersQueueGroup(db, pathId, pqGroup); context.SS->PersistRemovePersQueueGroupAlter(db, pathId); - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -1373,9 +1373,9 @@ public: } context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -1398,184 +1398,184 @@ public: } -namespace NCdcStreamState { - -class TConfigurePartsAtTable: public TSubOperationState { +namespace NCdcStreamState { + +class TConfigurePartsAtTable: public TSubOperationState { TString DebugHint() const override { - return TStringBuilder() - << "NCdcStreamState::TConfigurePartsAtTable" - << " operationId: " << OperationId; - } - - static bool IsExpectedTxType(TTxState::ETxType txType) { - switch (txType) { - case TTxState::TxCreateCdcStreamAtTable: - case TTxState::TxAlterCdcStreamAtTable: - case TTxState::TxDropCdcStreamAtTable: - return true; - default: - return false; - } - } - -protected: - virtual void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const = 0; - -public: - explicit TConfigurePartsAtTable(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " ProgressState" - << ", at schemeshard: " << context.SS->TabletID()); - - auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(IsExpectedTxType(txState->TxType)); - const auto& pathId = txState->TargetPathId; - - if (NTableState::CheckPartitioningChangedForTableModification(*txState, context)) { - NTableState::UpdatePartitioningForTableModification(OperationId, *txState, context); - } - - NKikimrTxDataShard::TFlatSchemeTransaction tx; - context.SS->FillSeqNo(tx, context.SS->StartRound(*txState)); - FillNotice(pathId, tx, context); - - TString txBody; + return TStringBuilder() + << "NCdcStreamState::TConfigurePartsAtTable" + << " operationId: " << OperationId; + } + + static bool IsExpectedTxType(TTxState::ETxType txType) { + switch (txType) { + case TTxState::TxCreateCdcStreamAtTable: + case TTxState::TxAlterCdcStreamAtTable: + case TTxState::TxDropCdcStreamAtTable: + return true; + default: + return false; + } + } + +protected: + virtual void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const = 0; + +public: + explicit TConfigurePartsAtTable(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " ProgressState" + << ", at schemeshard: " << context.SS->TabletID()); + + auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(IsExpectedTxType(txState->TxType)); + const auto& pathId = txState->TargetPathId; + + if (NTableState::CheckPartitioningChangedForTableModification(*txState, context)) { + NTableState::UpdatePartitioningForTableModification(OperationId, *txState, context); + } + + NKikimrTxDataShard::TFlatSchemeTransaction tx; + context.SS->FillSeqNo(tx, context.SS->StartRound(*txState)); + FillNotice(pathId, tx, context); + + TString txBody; Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); - - txState->ClearShardsInProgress(); - Y_VERIFY(txState->Shards.size()); - - for (ui32 i = 0; i < txState->Shards.size(); ++i) { - const auto& idx = txState->Shards[i].Idx; - const auto datashardId = context.SS->ShardInfos[idx].TabletID; - - auto ev = MakeHolder<TEvDataShard::TEvProposeTransaction>( - NKikimrTxDataShard::TX_KIND_SCHEME, context.SS->TabletID(), context.Ctx.SelfID, - ui64(OperationId.GetTxId()), txBody, - context.SS->SelectProcessingPrarams(pathId) - ); - - context.OnComplete.BindMsgToPipe(OperationId, datashardId, idx, ev.Release()); - } - - txState->UpdateShardsInProgress(TTxState::ConfigureParts); - return false; - } - - bool HandleReply(TEvDataShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply " << ev->Get()->ToString() - << ", at schemeshard: " << context.SS->TabletID()); - - if (!NTableState::CollectProposeTransactionResults(OperationId, ev, context)) { - return false; - } - - return true; - } - -private: - const TOperationId OperationId; - -}; // TConfigurePartsAtTable - -class TProposeAtTable: public TSubOperationState { + + txState->ClearShardsInProgress(); + Y_VERIFY(txState->Shards.size()); + + for (ui32 i = 0; i < txState->Shards.size(); ++i) { + const auto& idx = txState->Shards[i].Idx; + const auto datashardId = context.SS->ShardInfos[idx].TabletID; + + auto ev = MakeHolder<TEvDataShard::TEvProposeTransaction>( + NKikimrTxDataShard::TX_KIND_SCHEME, context.SS->TabletID(), context.Ctx.SelfID, + ui64(OperationId.GetTxId()), txBody, + context.SS->SelectProcessingPrarams(pathId) + ); + + context.OnComplete.BindMsgToPipe(OperationId, datashardId, idx, ev.Release()); + } + + txState->UpdateShardsInProgress(TTxState::ConfigureParts); + return false; + } + + bool HandleReply(TEvDataShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " HandleReply " << ev->Get()->ToString() + << ", at schemeshard: " << context.SS->TabletID()); + + if (!NTableState::CollectProposeTransactionResults(OperationId, ev, context)) { + return false; + } + + return true; + } + +private: + const TOperationId OperationId; + +}; // TConfigurePartsAtTable + +class TProposeAtTable: public TSubOperationState { TString DebugHint() const override { - return TStringBuilder() - << "NCdcStreamState::TProposeAtTable" - << " operationId: " << OperationId; - } - - static bool IsExpectedTxType(TTxState::ETxType txType) { - switch (txType) { - case TTxState::TxCreateCdcStreamAtTable: - case TTxState::TxAlterCdcStreamAtTable: - case TTxState::TxDropCdcStreamAtTable: - return true; - default: - return false; - } - } - -public: - explicit TProposeAtTable(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {TEvDataShard::TEvProposeTransactionResult::EventType}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " ProgressState" - << ", at schemeshard: " << context.SS->TabletID()); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(IsExpectedTxType(txState->TxType)); - - TSet<TTabletId> shardSet; - for (const auto& shard : txState->Shards) { - Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); - shardSet.insert(context.SS->ShardInfos.at(shard.Idx).TabletID); - } - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, txState->MinStep, shardSet); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvOperationPlan" - << ", step: " << ev->Get()->StepId - << ", at schemeshard: " << context.SS->TabletID()); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(IsExpectedTxType(txState->TxType)); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - auto table = context.SS->Tables.at(pathId); - - table->AlterVersion += 1; - - NIceDb::TNiceDb db(context.Txc.DB); - context.SS->PersistTableAlterVersion(db, pathId, table); - - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::ProposedWaitParts); - return true; - } - - bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " TEvDataShard::TEvSchemaChanged" - << " triggers early, save it" - << ", at schemeshard: " << context.SS->TabletID()); - - NTableState::CollectSchemaChanged(OperationId, ev, context); - return false; - } - -private: - const TOperationId OperationId; - -}; // TProposeAtTable - -} // NCdcStreamState - + return TStringBuilder() + << "NCdcStreamState::TProposeAtTable" + << " operationId: " << OperationId; + } + + static bool IsExpectedTxType(TTxState::ETxType txType) { + switch (txType) { + case TTxState::TxCreateCdcStreamAtTable: + case TTxState::TxAlterCdcStreamAtTable: + case TTxState::TxDropCdcStreamAtTable: + return true; + default: + return false; + } + } + +public: + explicit TProposeAtTable(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {TEvDataShard::TEvProposeTransactionResult::EventType}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " ProgressState" + << ", at schemeshard: " << context.SS->TabletID()); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(IsExpectedTxType(txState->TxType)); + + TSet<TTabletId> shardSet; + for (const auto& shard : txState->Shards) { + Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); + shardSet.insert(context.SS->ShardInfos.at(shard.Idx).TabletID); + } + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, txState->MinStep, shardSet); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " HandleReply TEvOperationPlan" + << ", step: " << ev->Get()->StepId + << ", at schemeshard: " << context.SS->TabletID()); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(IsExpectedTxType(txState->TxType)); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + auto table = context.SS->Tables.at(pathId); + + table->AlterVersion += 1; + + NIceDb::TNiceDb db(context.Txc.DB); + context.SS->PersistTableAlterVersion(db, pathId, table); + + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedWaitParts); + return true; + } + + bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " TEvDataShard::TEvSchemaChanged" + << " triggers early, save it" + << ", at schemeshard: " << context.SS->TabletID()); + + NTableState::CollectSchemaChanged(OperationId, ev, context); + return false; + } + +private: + const TOperationId OperationId; + +}; // TProposeAtTable + +} // NCdcStreamState + namespace NForceDrop { void ValidateNoTrasactionOnPathes(TOperationId operationId, const THashSet<TPathId>& pathes, TOperationContext& context); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index 8ad7fca2f66..4cc467dedc2 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -15,10 +15,10 @@ NKikimrSchemeOp::TModifyScheme CopyTableTask(NKikimr::NSchemeShard::TPath& src, scheme.SetFailOnExist(true); auto operation = scheme.MutableCreateTable(); - operation->SetName(dst.LeafName()); - operation->SetCopyFromTable(src.PathString()); + operation->SetName(dst.LeafName()); + operation->SetCopyFromTable(src.PathString()); operation->SetOmitFollowers(omitFollowers); - operation->SetIsBackup(isBackup); + operation->SetIsBackup(isBackup); return scheme; } @@ -49,17 +49,17 @@ NKikimrSchemeOp::TModifyScheme CreateIndexTask(NKikimr::NSchemeShard::TTableInde namespace NKikimr { namespace NSchemeShard { -TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { +TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables); - auto consistentCopying = tx.GetCreateConsistentCopyTables(); + auto consistentCopying = tx.GetCreateConsistentCopyTables(); if (0 == consistentCopying.CopyTableDescriptionsSize()) { TString msg = TStringBuilder() << "no task to do, empty list CopyTableDescriptions"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; } - TPath firstPath = TPath::Resolve(consistentCopying.GetCopyTableDescriptions(0).GetSrcPath(), context.SS); + TPath firstPath = TPath::Resolve(consistentCopying.GetCopyTableDescriptions(0).GetSrcPath(), context.SS); { auto checks = TPath::TChecker(firstPath); checks @@ -92,7 +92,7 @@ TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId, TVector<ISubOperationBase::TPtr> result; - for (auto& descr: consistentCopying.GetCopyTableDescriptions()) { + for (auto& descr: consistentCopying.GetCopyTableDescriptions()) { auto& srcStr = descr.GetSrcPath(); auto& dstStr = descr.GetDstPath(); @@ -124,33 +124,33 @@ TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId, continue; } - for (const auto& child: srcPath.Base()->GetChildren()) { - const auto& name = child.first; - const auto& pathId = child.second; + for (const auto& child: srcPath.Base()->GetChildren()) { + const auto& name = child.first; + const auto& pathId = child.second; TPath srcIndexPath = srcPath.Child(name); TPath dstIndexPath = dstPath.Child(name); - + if (srcIndexPath.IsDeleted()) { continue; } - if (!srcIndexPath.IsTableIndex()) { - continue; - } - - Y_VERIFY(srcIndexPath.Base()->PathId == pathId); + if (!srcIndexPath.IsTableIndex()) { + continue; + } + + Y_VERIFY(srcIndexPath.Base()->PathId == pathId); Y_VERIFY_S(srcIndexPath.Base()->GetChildren().size() == 1, srcIndexPath.PathString() << " has children " << srcIndexPath.Base()->GetChildren().size() << " but 1 exected"); TTableIndexInfo::TPtr indexInfo = context.SS->Indexes.at(pathId); - result.push_back(CreateNewTableIndex(NextPartId(nextId, result), CreateIndexTask(indexInfo, dstIndexPath))); + result.push_back(CreateNewTableIndex(NextPartId(nextId, result), CreateIndexTask(indexInfo, dstIndexPath))); TString srcImplTableName = srcIndexPath.Base()->GetChildren().begin()->first; TPath srcImplTable = srcIndexPath.Child(srcImplTableName); - Y_VERIFY(srcImplTable.Base()->PathId == srcIndexPath.Base()->GetChildren().begin()->second); + Y_VERIFY(srcImplTable.Base()->PathId == srcIndexPath.Base()->GetChildren().begin()->second); TPath dstImplTable = dstIndexPath.Child(srcImplTableName); - result.push_back(CreateCopyTable(NextPartId(nextId, result), + result.push_back(CreateCopyTable(NextPartId(nextId, result), CopyTableTask(srcImplTable, dstImplTable, descr.GetOmitFollowers(), descr.GetIsBackup()))); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index cdaad34303c..f504c34ef58 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -200,8 +200,8 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedWaitParts); return true; } @@ -231,7 +231,7 @@ public: class TCopyTable: public TSubOperation { const TOperationId OperationId; - const TTxTransaction Transaction; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -283,11 +283,11 @@ class TCopyTable: public TSubOperation { } public: - TCopyTable(TOperationId id, const TTxTransaction& tx) + TCopyTable(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCopyTable(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -305,7 +305,7 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const TString& parentPath = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetCreateTable().GetName(); + const TString& name = Transaction.GetCreateTable().GetName(); const auto acceptExisted = !Transaction.GetFailOnExist(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -319,8 +319,8 @@ public: TPath parent = TPath::Resolve(parentPath, context.SS); { TPath::TChecker checks = parent.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -329,13 +329,13 @@ public: if (checks) { if (parent.Base()->IsTableIndex()) { - checks - .IsInsideTableIndexPath() + checks + .IsInsideTableIndexPath() .IsUnderCreating(NKikimrScheme::StatusNameConflict) .IsUnderTheSameOperation(OperationId.GetTxId()); //allow only as part of copying base table } else { - checks - .NotUnderOperation() + checks + .NotUnderOperation() .IsCommonSensePath() .IsLikeDirectory(); } @@ -350,11 +350,11 @@ public: } } - TPath srcPath = TPath::Resolve(Transaction.GetCreateTable().GetCopyFromTable(), context.SS); + TPath srcPath = TPath::Resolve(Transaction.GetCreateTable().GetCopyFromTable(), context.SS); { TPath::TChecker checks = srcPath.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .IsResolved() .NotDeleted() .NotUnderDeleting() @@ -364,8 +364,8 @@ public: if (checks) { if (parent.Base()->IsTableIndex()) { - checks - .IsInsideTableIndexPath() //copy imp index table as index index table, not a separate one + checks + .IsInsideTableIndexPath() //copy imp index table as index index table, not a separate one .NotChildren(); //imp table doesnt have indexes } else { checks.IsCommonSensePath(); @@ -382,19 +382,19 @@ public: } const ui32 maxShardsToCreate = srcPath.Shards(); - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + TPath dstPath = parent.Child(name); { TPath::TChecker checks = dstPath.Check(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeTable, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -409,8 +409,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit(maxShardsToCreate) - .PathShardsLimit(maxShardsToCreate) - .IsValidACL(acl); + .PathShardsLimit(maxShardsToCreate) + .IsValidACL(acl); } if (!checks) { @@ -422,20 +422,20 @@ public: } } - auto domainInfo = parent.DomainInfo(); - bool transactionSupport = domainInfo->IsSupportTransactions(); - if (domainInfo->GetAlter()) { - TPathId domainId = parent.DomainId(); - Y_VERIFY(context.SS->PathsById.contains(domainId)); - TPathElement::TPtr domain = context.SS->PathsById.at(domainId); - Y_VERIFY(domain->PlannedToCreate() || domain->HasActiveChanges()); + auto domainInfo = parent.DomainInfo(); + bool transactionSupport = domainInfo->IsSupportTransactions(); + if (domainInfo->GetAlter()) { + TPathId domainId = parent.DomainId(); + Y_VERIFY(context.SS->PathsById.contains(domainId)); + TPathElement::TPtr domain = context.SS->PathsById.at(domainId); + Y_VERIFY(domain->PlannedToCreate() || domain->HasActiveChanges()); - transactionSupport |= domainInfo->GetAlter()->IsSupportTransactions(); + transactionSupport |= domainInfo->GetAlter()->IsSupportTransactions(); } - if (!transactionSupport) { + if (!transactionSupport) { result->SetError(NKikimrScheme::StatusNameConflict, "Inclusive subDomian do not support shared transactions"); - return result; - } + return result; + } TString errStr; @@ -465,14 +465,14 @@ public: } } - auto schema = Transaction.GetCreateTable(); + auto schema = Transaction.GetCreateTable(); const bool omitFollowers = schema.GetOmitFollowers(); - const bool isBackup = schema.GetIsBackup(); + const bool isBackup = schema.GetIsBackup(); PrepareScheme(&schema, name, srcTableInfo, context); if (omitFollowers) { schema.MutablePartitionConfig()->AddFollowerGroups()->Clear(); - } - schema.SetIsBackup(isBackup); + } + schema.SetIsBackup(isBackup); NKikimrSchemeOp::TPartitionConfig compilationPartitionConfig; if (!TPartitionConfigMerger::ApplyChanges(compilationPartitionConfig, srcTableInfo->PartitionConfig(), schema.GetPartitionConfig(), AppData(), errStr) @@ -483,8 +483,8 @@ public: schema.MutablePartitionConfig()->CopyFrom(compilationPartitionConfig); const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; - const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr); + const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr); if (!alterData.Get()) { result->SetError(NKikimrScheme::StatusSchemeError, errStr); return result; @@ -541,10 +541,10 @@ public: context.DbChanges.PersistTable(allocatedPathId); context.DbChanges.PersistTxState(OperationId); - // copy attrs without any checks - TUserAttributes::TPtr userAttrs = new TUserAttributes(1); - userAttrs->Attrs = srcPath.Base()->UserAttrs->Attrs; - + // copy attrs without any checks + TUserAttributes::TPtr userAttrs = new TUserAttributes(1); + userAttrs->Attrs = srcPath.Base()->UserAttrs->Attrs; + dstPath.MaterializeLeaf(owner, allocatedPathId); result->SetPathId(dstPath.Base()->PathId.LocalPathId); @@ -553,7 +553,7 @@ public: newTable->LastTxId = OperationId.GetTxId(); newTable->PathState = TPathElement::EPathState::EPathStateCreate; newTable->PathType = TPathElement::EPathType::EPathTypeTable; - newTable->UserAttrs->AlterData = userAttrs; + newTable->UserAttrs->AlterData = userAttrs; srcPath.Base()->PathState = TPathElement::EPathState::EPathStateCopying; srcPath.Base()->LastTxId = OperationId.GetTxId(); @@ -623,14 +623,14 @@ public: "TCopyTable AbortPropose" << ", opId: " << OperationId << ", at schemeshard: " << context.SS->TabletID()); - } - + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCopyTable AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -652,16 +652,16 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateCopyTable(TOperationId id, const TTxTransaction& tx) { - return new TCopyTable(id, tx); -} - +ISubOperationBase::TPtr CreateCopyTable(TOperationId id, const TTxTransaction& tx) { + return new TCopyTable(id, tx); +} + ISubOperationBase::TPtr CreateCopyTable(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state != TTxState::Invalid); return new TCopyTable(id, state); } -TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { +TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); auto copying = tx.GetCreateTable(); @@ -700,7 +700,7 @@ TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxT operation->SetName(copying.GetName()); operation->SetCopyFromTable(copying.GetCopyFromTable()); operation->SetOmitFollowers(copying.GetOmitFollowers()); - operation->SetIsBackup(copying.GetIsBackup()); + operation->SetIsBackup(copying.GetIsBackup()); operation->MutablePartitionConfig()->CopyFrom(copying.GetPartitionConfig()); result.push_back(CreateCopyTable(NextPartId(nextId, result), schema)); @@ -710,13 +710,13 @@ TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxT auto name = child.first; auto pathId = child.second; - TPath childPath = srcPath.Child(name); - if (!childPath.IsTableIndex() || childPath.IsDeleted()) { + TPath childPath = srcPath.Child(name); + if (!childPath.IsTableIndex() || childPath.IsDeleted()) { continue; } - Y_VERIFY(childPath.Base()->PathId == pathId); - Y_VERIFY_S(childPath.Base()->GetChildren().size() == 1, childPath.PathString() << " has children " << childPath.Base()->GetChildren().size()); + Y_VERIFY(childPath.Base()->PathId == pathId); + Y_VERIFY_S(childPath.Base()->GetChildren().size() == 1, childPath.PathString() << " has children " << childPath.Base()->GetChildren().size()); TTableIndexInfo::TPtr indexInfo = context.SS->Indexes.at(pathId); { @@ -732,13 +732,13 @@ TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxT for (const auto& dataColumn: indexInfo->IndexDataColumns) { *operation->MutableDataColumnNames()->Add() = dataColumn; } - + result.push_back(CreateNewTableIndex(NextPartId(nextId, result), schema)); } - TString implTableName = childPath.Base()->GetChildren().begin()->first; - TPath implTable = childPath.Child(implTableName); - Y_VERIFY(implTable.Base()->PathId == childPath.Base()->GetChildren().begin()->second); + TString implTableName = childPath.Base()->GetChildren().begin()->first; + TPath implTable = childPath.Child(implTableName); + Y_VERIFY(implTable.Base()->PathId == childPath.Base()->GetChildren().begin()->second); { NKikimrSchemeOp::TModifyScheme schema; @@ -750,7 +750,7 @@ TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxT operation->SetName(implTableName); operation->SetCopyFromTable(implTable.PathString()); operation->SetOmitFollowers(copying.GetOmitFollowers()); - operation->SetIsBackup(copying.GetIsBackup()); + operation->SetIsBackup(copying.GetIsBackup()); result.push_back(CreateCopyTable(NextPartId(nextId, result), schema)); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp index 998d18a8ccf..4c0c967af0f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp @@ -1,119 +1,119 @@ -#include "schemeshard__operation_backup_restore_common.h" -#include "schemeshard_billing_helpers.h" +#include "schemeshard__operation_backup_restore_common.h" +#include "schemeshard_billing_helpers.h" -namespace NKikimr { +namespace NKikimr { namespace NSchemeShard { -struct TBackup { - static constexpr TStringBuf Name() { - return "TBackup"; +struct TBackup { + static constexpr TStringBuf Name() { + return "TBackup"; } - static bool HasTask(const TTxTransaction& tx) { - return tx.HasBackup(); + static bool HasTask(const TTxTransaction& tx) { + return tx.HasBackup(); } - static TString GetTableName(const TTxTransaction& tx) { - return tx.GetBackup().GetTableName(); + static TString GetTableName(const TTxTransaction& tx) { + return tx.GetBackup().GetTableName(); } - static void ProposeTx(const TOperationId& opId, TTxState& txState, TOperationContext& context) { - auto seqNo = context.SS->StartRound(txState); - const auto& processingParams = context.SS->SelectProcessingPrarams(txState.TargetPathId); + static void ProposeTx(const TOperationId& opId, TTxState& txState, TOperationContext& context) { + auto seqNo = context.SS->StartRound(txState); + const auto& processingParams = context.SS->SelectProcessingPrarams(txState.TargetPathId); - Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); - TTableInfo::TPtr table = context.SS->Tables.at(txState.TargetPathId); + Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); + TTableInfo::TPtr table = context.SS->Tables.at(txState.TargetPathId); NKikimrSchemeOp::TBackupTask backup = table->BackupSettings; - - for (ui32 i = 0; i < txState.Shards.size(); ++i) { - auto idx = txState.Shards[i].Idx; + + for (ui32 i = 0; i < txState.Shards.size(); ++i) { + auto idx = txState.Shards[i].Idx; auto datashardId = context.SS->ShardInfos[idx].TabletID; LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Propose backup" << " to datashard " << datashardId - << " txid " << opId - << " at schemeshard " << context.SS->SelfTabletId()); + << " txid " << opId + << " at schemeshard " << context.SS->SelfTabletId()); - TString txBody = context.SS->FillBackupTxBody(txState.TargetPathId, backup, i, seqNo); + TString txBody = context.SS->FillBackupTxBody(txState.TargetPathId, backup, i, seqNo); THolder<TEvDataShard::TEvProposeTransaction> event = THolder(new TEvDataShard::TEvProposeTransaction(NKikimrTxDataShard::TX_KIND_SCHEME, context.SS->TabletID(), context.Ctx.SelfID, - ui64(opId.GetTxId()), + ui64(opId.GetTxId()), txBody, processingParams)); - context.OnComplete.BindMsgToPipe(opId, datashardId, idx, event.Release()); - - backup.ClearTable(); + context.OnComplete.BindMsgToPipe(opId, datashardId, idx, event.Release()); + + backup.ClearTable(); } } - static ui64 RequestUnits(ui64 bytes, ui64 rows) { - Y_UNUSED(rows); - return TRUCalculator::ReadTable(bytes); - } - - static void FinishStats(const TOperationId& opId, TTxState& txState, TOperationContext& context) { - if (txState.TxType != TTxState::TxBackup) { + static ui64 RequestUnits(ui64 bytes, ui64 rows) { + Y_UNUSED(rows); + return TRUCalculator::ReadTable(bytes); + } + + static void FinishStats(const TOperationId& opId, TTxState& txState, TOperationContext& context) { + if (txState.TxType != TTxState::TxBackup) { return; } - Y_VERIFY(TAppData::TimeProvider.Get() != nullptr); - const ui64 ts = TAppData::TimeProvider->Now().Seconds(); + Y_VERIFY(TAppData::TimeProvider.Get() != nullptr); + const ui64 ts = TAppData::TimeProvider->Now().Seconds(); - Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); - TTableInfo::TPtr table = context.SS->Tables[txState.TargetPathId]; + Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); + TTableInfo::TPtr table = context.SS->Tables[txState.TargetPathId]; - auto& backupInfo = table->BackupHistory[opId.GetTxId()]; + auto& backupInfo = table->BackupHistory[opId.GetTxId()]; - backupInfo.StartDateTime = txState.StartTime.Seconds(); - backupInfo.CompletionDateTime = ts; - backupInfo.TotalShardCount = table->GetPartitions().size(); - backupInfo.SuccessShardCount = CountIf(txState.ShardStatuses, [](const auto& kv) { - return kv.second.Success; - }); - backupInfo.ShardStatuses = std::move(txState.ShardStatuses); - backupInfo.DataTotalSize = txState.DataTotalSize; + backupInfo.StartDateTime = txState.StartTime.Seconds(); + backupInfo.CompletionDateTime = ts; + backupInfo.TotalShardCount = table->GetPartitions().size(); + backupInfo.SuccessShardCount = CountIf(txState.ShardStatuses, [](const auto& kv) { + return kv.second.Success; + }); + backupInfo.ShardStatuses = std::move(txState.ShardStatuses); + backupInfo.DataTotalSize = txState.DataTotalSize; NIceDb::TNiceDb db(context.Txc.DB); - context.SS->PersistCompletedBackup(db, opId.GetTxId(), txState, backupInfo); + context.SS->PersistCompletedBackup(db, opId.GetTxId(), txState, backupInfo); } - static void PersistTask(const TPathId& pathId, const TTxTransaction& tx, TOperationContext& context) { - Y_VERIFY(context.SS->Tables.contains(pathId)); - TTableInfo::TPtr table = context.SS->Tables.at(pathId); + static void PersistTask(const TPathId& pathId, const TTxTransaction& tx, TOperationContext& context) { + Y_VERIFY(context.SS->Tables.contains(pathId)); + TTableInfo::TPtr table = context.SS->Tables.at(pathId); - table->BackupSettings = tx.GetBackup(); + table->BackupSettings = tx.GetBackup(); NIceDb::TNiceDb db(context.Txc.DB); - context.SS->PersistBackupSettings(db, pathId, table->BackupSettings); + context.SS->PersistBackupSettings(db, pathId, table->BackupSettings); } - static void PersistDone(const TPathId& pathId, TOperationContext& context) { + static void PersistDone(const TPathId& pathId, TOperationContext& context) { NIceDb::TNiceDb db(context.Txc.DB); context.SS->PersistBackupDone(db, pathId); } - - static bool NeedToBill(const TPathId& pathId, TOperationContext& context) { - Y_VERIFY(context.SS->Tables.contains(pathId)); - auto table = context.SS->Tables.at(pathId); - return table->BackupSettings.GetNeedToBill(); - } + + static bool NeedToBill(const TPathId& pathId, TOperationContext& context) { + Y_VERIFY(context.SS->Tables.contains(pathId)); + auto table = context.SS->Tables.at(pathId); + return table->BackupSettings.GetNeedToBill(); + } }; -ISubOperationBase::TPtr CreateBackup(TOperationId id, const TTxTransaction& tx) { - return new TBackupRestoreOperationBase<TBackup, TEvDataShard::TEvCancelBackup>( - TTxState::TxBackup, TPathElement::EPathState::EPathStateBackup, id, tx - ); -} - -ISubOperationBase::TPtr CreateBackup(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); - return new TBackupRestoreOperationBase<TBackup, TEvDataShard::TEvCancelBackup>( - TTxState::TxBackup, TPathElement::EPathState::EPathStateBackup, id, state - ); +ISubOperationBase::TPtr CreateBackup(TOperationId id, const TTxTransaction& tx) { + return new TBackupRestoreOperationBase<TBackup, TEvDataShard::TEvCancelBackup>( + TTxState::TxBackup, TPathElement::EPathState::EPathStateBackup, id, tx + ); +} + +ISubOperationBase::TPtr CreateBackup(TOperationId id, TTxState::ETxState state) { + Y_VERIFY(state != TTxState::Invalid); + return new TBackupRestoreOperationBase<TBackup, TEvDataShard::TEvCancelBackup>( + TTxState::TxBackup, TPathElement::EPathState::EPathStateBackup, id, state + ); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp index d8883117a70..b32fb7d9692 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp @@ -146,8 +146,8 @@ TTxState& PrepareChanges(TOperationId operationId, TPathElement::TPtr parentDir, class TCreateBlockStoreVolume: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -161,8 +161,8 @@ class TCreateBlockStoreVolume: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -178,7 +178,7 @@ class TCreateBlockStoreVolume: public TSubOperation { return MakeHolder<NBSVState::TConfigureParts>(OperationId); case TTxState::Propose: return MakeHolder<NBSVState::TPropose>(OperationId); - case TTxState::Done: + case TTxState::Done: return MakeHolder<TDone>(OperationId); default: return nullptr; @@ -195,11 +195,11 @@ class TCreateBlockStoreVolume: public TSubOperation { } public: - TCreateBlockStoreVolume(TOperationId id, const TTxTransaction& tx) + TCreateBlockStoreVolume(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateBlockStoreVolume(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -211,11 +211,11 @@ public: THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& operation = Transaction.GetCreateBlockStoreVolume(); + const auto& operation = Transaction.GetCreateBlockStoreVolume(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetCreateBlockStoreVolume().GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetCreateBlockStoreVolume().GetName(); const auto defaultPartitionCount = TBlockStoreVolumeInfo::CalculateDefaultPartitionCount( operation.GetVolumeConfig()); @@ -223,7 +223,7 @@ public: LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateBlockStoreVolume Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", opId: " << OperationId << ", at schemeshard: " << ssId); @@ -251,20 +251,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeBlockStoreVolume, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -275,8 +275,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit(shardsToCreate) - .PathShardsLimit(shardsToCreate) - .IsValidACL(acl); + .PathShardsLimit(shardsToCreate) + .IsValidACL(acl); } if (!checks) { @@ -292,9 +292,9 @@ public: } } - TString errStr; - - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -401,16 +401,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateBlockStoreVolume"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateBlockStoreVolume"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateBlockStoreVolume AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -421,12 +421,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateNewBSV(TOperationId id, const TTxTransaction& tx) { - return new TCreateBlockStoreVolume(id, tx); -} - +ISubOperationBase::TPtr CreateNewBSV(TOperationId id, const TTxTransaction& tx) { + return new TCreateBlockStoreVolume(id, tx); +} + ISubOperationBase::TPtr CreateNewBSV(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateBlockStoreVolume(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp index 9a870db85b4..2d957165c6a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp @@ -43,7 +43,7 @@ TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId nextId, const TTx } tableIndexCreation.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly); - result.push_back(CreateNewTableIndex(NextPartId(nextId, result), tableIndexCreation)); + result.push_back(CreateNewTableIndex(NextPartId(nextId, result), tableIndexCreation)); } { @@ -52,7 +52,7 @@ TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId nextId, const TTx auto& shapshot = *initialize.MutableInitiateBuildIndexMainTable(); shapshot.SetTableName(table.LeafName()); - result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(nextId, result), initialize)); + result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(nextId, result), initialize)); } { @@ -75,7 +75,7 @@ TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId nextId, const TTx indexImplTableDescription.MutablePartitionConfig()->MutableCompactionPolicy()->SetKeepEraseMarkers(true); indexImplTableDescription.MutablePartitionConfig()->SetShadowData(true); - result.push_back(CreateInitializeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableCreation)); + result.push_back(CreateInitializeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableCreation)); } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp index 3c0c92c00d0..01e4cd9df82 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp @@ -1,678 +1,678 @@ -#include "schemeshard__operation_part.h" -#include "schemeshard__operation_common.h" -#include "schemeshard_impl.h" - +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/persqueue/writer/source_id_encoding.h> - -#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) - -namespace NKikimr { + +#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr { namespace NSchemeShard { - -namespace { - -class TPropose: public TSubOperationState { + +namespace { + +class TPropose: public TSubOperationState { TString DebugHint() const override { - return TStringBuilder() - << "NewCdcStream TPropose" - << " opId# " << OperationId << " "; - } - -public: - explicit TPropose(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateCdcStream); - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - const auto step = TStepId(ev->Get()->StepId); - - LOG_I(DebugHint() << "HandleReply TEvOperationPlan" - << ": step# " << step); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateCdcStream); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->CdcStreams.contains(pathId)); - auto stream = context.SS->CdcStreams.at(pathId); - - NIceDb::TNiceDb db(context.Txc.DB); - - path->StepCreated = step; - context.SS->PersistCreateStep(db, pathId, step); - - context.SS->PersistCdcStream(db, pathId); - context.SS->CdcStreams[pathId] = stream->AlterData; - - context.SS->TabletCounters->Simple()[COUNTER_CDC_STREAMS_COUNT].Add(1); - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - return true; - } - -private: - const TOperationId OperationId; - -}; // TPropose - -class TNewCdcStream: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::Propose; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return TTxState::Done; - default: - return TTxState::Invalid; - } - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return THolder(new TPropose(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TNewCdcStream(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TNewCdcStream(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + return TStringBuilder() + << "NewCdcStream TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateCdcStream); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateCdcStream); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->CdcStreams.contains(pathId)); + auto stream = context.SS->CdcStreams.at(pathId); + + NIceDb::TNiceDb db(context.Txc.DB); + + path->StepCreated = step; + context.SS->PersistCreateStep(db, pathId, step); + + context.SS->PersistCdcStream(db, pathId); + context.SS->CdcStreams[pathId] = stream->AlterData; + + context.SS->TabletCounters->Simple()[COUNTER_CDC_STREAMS_COUNT].Add(1); + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TNewCdcStream: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return THolder(new TPropose(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TNewCdcStream(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TNewCdcStream(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetCreateCdcStream(); - const auto& streamDesc = op.GetStreamDescription(); - const auto& streamName = streamDesc.GetName(); + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetCreateCdcStream(); + const auto& streamDesc = op.GetStreamDescription(); + const auto& streamName = streamDesc.GetName(); const auto acceptExisted = !Transaction.GetFailOnExist(); - - LOG_N("TNewCdcStream Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << streamName); - + + LOG_N("TNewCdcStream Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto tablePath = TPath::Resolve(workingDir, context.SS); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .IsAtLocalSchemeShard(); - - if (streamPath.IsResolved()) { - checks - .IsResolved() - .NotUnderDeleting() - .FailOnExist(TPathElement::EPathType::EPathTypeCdcStream, acceptExisted); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (checks) { - checks - .IsValidLeafName() - .PathsLimit() - .DirChildrenLimit(); - } - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - if (streamPath.IsResolved()) { - result->SetPathCreateTxId(ui64(streamPath.Base()->CreateTxId)); - result->SetPathId(streamPath.Base()->PathId.LocalPathId); - } - return result; - } - } - - switch (streamDesc.GetMode()) { + + const auto tablePath = TPath::Resolve(workingDir, context.SS); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .IsAtLocalSchemeShard(); + + if (streamPath.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeCdcStream, acceptExisted); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .PathsLimit() + .DirChildrenLimit(); + } + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + if (streamPath.IsResolved()) { + result->SetPathCreateTxId(ui64(streamPath.Base()->CreateTxId)); + result->SetPathId(streamPath.Base()->PathId.LocalPathId); + } + return result; + } + } + + switch (streamDesc.GetMode()) { case NKikimrSchemeOp::ECdcStreamModeKeysOnly: case NKikimrSchemeOp::ECdcStreamModeUpdate: case NKikimrSchemeOp::ECdcStreamModeNewImage: case NKikimrSchemeOp::ECdcStreamModeOldImage: case NKikimrSchemeOp::ECdcStreamModeNewAndOldImages: - break; - default: + break; + default: result->SetError(NKikimrScheme::StatusInvalidParameter, TStringBuilder() - << "Invalid stream mode: " << static_cast<ui32>(streamDesc.GetMode())); - return result; - } - - TString errStr; - if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { + << "Invalid stream mode: " << static_cast<ui32>(streamDesc.GetMode())); + return result; + } + + TString errStr; + if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); - return result; - } - - auto stream = TCdcStreamInfo::Create(streamDesc); - Y_VERIFY(stream); - - const auto pathId = context.SS->AllocatePathId(); - context.MemChanges.GrabNewPath(context.SS, pathId); - context.MemChanges.GrabPath(context.SS, tablePath.Base()->PathId); - context.MemChanges.GrabNewTxState(context.SS, OperationId); - context.MemChanges.GrabDomain(context.SS, streamPath.DomainId()); - context.MemChanges.GrabNewCdcStream(context.SS, pathId); - - context.DbChanges.PersistPath(pathId); - context.DbChanges.PersistPath(tablePath.Base()->PathId); - context.DbChanges.PersistAlterCdcStream(pathId); - context.DbChanges.PersistTxState(OperationId); - - streamPath.MaterializeLeaf(owner, pathId); - result->SetPathId(pathId.LocalPathId); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateCdcStream, streamPath.Base()->PathId); - txState.State = TTxState::Propose; - + return result; + } + + auto stream = TCdcStreamInfo::Create(streamDesc); + Y_VERIFY(stream); + + const auto pathId = context.SS->AllocatePathId(); + context.MemChanges.GrabNewPath(context.SS, pathId); + context.MemChanges.GrabPath(context.SS, tablePath.Base()->PathId); + context.MemChanges.GrabNewTxState(context.SS, OperationId); + context.MemChanges.GrabDomain(context.SS, streamPath.DomainId()); + context.MemChanges.GrabNewCdcStream(context.SS, pathId); + + context.DbChanges.PersistPath(pathId); + context.DbChanges.PersistPath(tablePath.Base()->PathId); + context.DbChanges.PersistAlterCdcStream(pathId); + context.DbChanges.PersistTxState(OperationId); + + streamPath.MaterializeLeaf(owner, pathId); + result->SetPathId(pathId.LocalPathId); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateCdcStream, streamPath.Base()->PathId); + txState.State = TTxState::Propose; + streamPath.Base()->PathState = NKikimrSchemeOp::EPathStateCreate; - streamPath.Base()->CreateTxId = OperationId.GetTxId(); - streamPath.Base()->LastTxId = OperationId.GetTxId(); - streamPath.Base()->PathType = TPathElement::EPathType::EPathTypeCdcStream; - - context.SS->CdcStreams[pathId] = stream; - context.SS->IncrementPathDbRefCount(pathId); - - streamPath.DomainInfo()->IncPathsInside(); - tablePath.Base()->IncAliveChildren(); - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TNewCdcStream"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TNewCdcStream AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TNewCdcStream - -class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { -protected: - void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - auto table = context.SS->Tables.at(pathId); - - auto& notice = *tx.MutableCreateCdcStreamNotice(); - PathIdFromPathId(pathId, notice.MutablePathId()); - notice.SetTableSchemaVersion(table->AlterVersion + 1); - - bool found = false; - for (const auto& [childName, childPathId] : path->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsCdcStream() || childPath->Dropped()) { - continue; - } - - Y_VERIFY(context.SS->CdcStreams.contains(childPathId)); - auto stream = context.SS->CdcStreams.at(childPathId); - - if (stream->State != TCdcStreamInfo::EState::ECdcStreamStateInvalid) { - continue; - } - - Y_VERIFY_S(!found, "Too many cdc streams are planned to create" - << ": found# " << PathIdFromPathId(notice.GetStreamDescription().GetPathId()) - << ", another# " << childPathId); - found = true; - - Y_VERIFY(stream->AlterData); - context.SS->DescribeCdcStream(childPathId, childName, stream->AlterData, *notice.MutableStreamDescription()); - } - } - -public: - using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; - -}; // TConfigurePartsAtTable - -class TNewCdcStreamAtTable: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::ConfigureParts; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return TTxState::Propose; - case TTxState::Propose: - return TTxState::ProposedWaitParts; - case TTxState::ProposedWaitParts: - return TTxState::Done; - default: - return TTxState::Invalid; - } - - return TTxState::Invalid; - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return THolder(new TConfigurePartsAtTable(OperationId)); - case TTxState::Propose: - return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); - case TTxState::ProposedWaitParts: - return THolder(new NTableState::TProposedWaitParts(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + streamPath.Base()->CreateTxId = OperationId.GetTxId(); + streamPath.Base()->LastTxId = OperationId.GetTxId(); + streamPath.Base()->PathType = TPathElement::EPathType::EPathTypeCdcStream; + + context.SS->CdcStreams[pathId] = stream; + context.SS->IncrementPathDbRefCount(pathId); + + streamPath.DomainInfo()->IncPathsInside(); + tablePath.Base()->IncAliveChildren(); + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TNewCdcStream"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TNewCdcStream AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TNewCdcStream + +class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { +protected: + void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + auto table = context.SS->Tables.at(pathId); + + auto& notice = *tx.MutableCreateCdcStreamNotice(); + PathIdFromPathId(pathId, notice.MutablePathId()); + notice.SetTableSchemaVersion(table->AlterVersion + 1); + + bool found = false; + for (const auto& [childName, childPathId] : path->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsCdcStream() || childPath->Dropped()) { + continue; + } + + Y_VERIFY(context.SS->CdcStreams.contains(childPathId)); + auto stream = context.SS->CdcStreams.at(childPathId); + + if (stream->State != TCdcStreamInfo::EState::ECdcStreamStateInvalid) { + continue; + } + + Y_VERIFY_S(!found, "Too many cdc streams are planned to create" + << ": found# " << PathIdFromPathId(notice.GetStreamDescription().GetPathId()) + << ", another# " << childPathId); + found = true; + + Y_VERIFY(stream->AlterData); + context.SS->DescribeCdcStream(childPathId, childName, stream->AlterData, *notice.MutableStreamDescription()); + } + } + +public: + using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; + +}; // TConfigurePartsAtTable + +class TNewCdcStreamAtTable: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::ConfigureParts; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return TTxState::Propose; + case TTxState::Propose: + return TTxState::ProposedWaitParts; + case TTxState::ProposedWaitParts: + return TTxState::Done; + default: + return TTxState::Invalid; + } + + return TTxState::Invalid; + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return THolder(new TConfigurePartsAtTable(OperationId)); + case TTxState::Propose: + return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); + case TTxState::ProposedWaitParts: + return THolder(new NTableState::TProposedWaitParts(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetCreateCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamName = op.GetStreamDescription().GetName(); - - LOG_N("TNewCdcStreamAtTable Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << tableName << "/" << streamName); - + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetCreateCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamName = op.GetStreamDescription().GetName(); + + LOG_N("TNewCdcStreamAtTable Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << tableName << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto workingDirPath = TPath::Resolve(workingDir, context.SS); - { - const auto checks = workingDirPath.Check(); - checks - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCommonSensePath() - .IsLikeDirectory() - .NotUnderDeleting(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << workingDirPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - const auto tablePath = workingDirPath.Child(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + + const auto workingDirPath = TPath::Resolve(workingDir, context.SS); + { + const auto checks = workingDirPath.Check(); + checks + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCommonSensePath() + .IsLikeDirectory() + .NotUnderDeleting(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << workingDirPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + const auto tablePath = workingDirPath.Child(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); - return result; - } - - context.DbChanges.PersistTxState(OperationId); - - Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); - auto table = context.SS->Tables.at(tablePath.Base()->PathId); - - Y_VERIFY(table->AlterVersion != 0); - Y_VERIFY(!table->AlterData); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateCdcStreamAtTable, tablePath.Base()->PathId); - txState.State = TTxState::ConfigureParts; - + return result; + } + + context.DbChanges.PersistTxState(OperationId); + + Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); + auto table = context.SS->Tables.at(tablePath.Base()->PathId); + + Y_VERIFY(table->AlterVersion != 0); + Y_VERIFY(!table->AlterData); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateCdcStreamAtTable, tablePath.Base()->PathId); + txState.State = TTxState::ConfigureParts; + tablePath.Base()->PathState = NKikimrSchemeOp::EPathStateAlter; - tablePath.Base()->LastTxId = OperationId.GetTxId(); - - for (const auto& splitOpId : table->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); - } - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TNewCdcStreamAtTable"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TNewCdcStreamAtTable AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TNewCdcStreamAtTable - -} // anonymous - -ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { - return new TNewCdcStream(id, tx); -} - -ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, TTxState::ETxState state) { - return new TNewCdcStream(id, state); -} - -ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { - return new TNewCdcStreamAtTable(id, tx); -} - -ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { - return new TNewCdcStreamAtTable(id, state); -} - -TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { + tablePath.Base()->LastTxId = OperationId.GetTxId(); + + for (const auto& splitOpId : table->GetSplitOpsInFlight()) { + context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); + } + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TNewCdcStreamAtTable"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TNewCdcStreamAtTable AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TNewCdcStreamAtTable + +} // anonymous + +ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { + return new TNewCdcStream(id, tx); +} + +ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, TTxState::ETxState state) { + return new TNewCdcStream(id, state); +} + +ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { + return new TNewCdcStreamAtTable(id, tx); +} + +ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { + return new TNewCdcStreamAtTable(id, state); +} + +TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream); - - LOG_D("CreateNewCdcStream" - << ": opId# " << opId - << ", tx# " << tx.ShortDebugString()); - + + LOG_D("CreateNewCdcStream" + << ": opId# " << opId + << ", tx# " << tx.ShortDebugString()); + const auto acceptExisted = !tx.GetFailOnExist(); - const auto& op = tx.GetCreateCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamDesc = op.GetStreamDescription(); - const auto& streamName = streamDesc.GetName(); - - const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); - - const auto tablePath = workingDirPath.Child(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - const auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .IsAtLocalSchemeShard(); - - if (streamPath.IsResolved()) { - checks - .IsResolved() - .NotUnderDeleting() + const auto& op = tx.GetCreateCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamDesc = op.GetStreamDescription(); + const auto& streamName = streamDesc.GetName(); + + const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); + + const auto tablePath = workingDirPath.Child(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + const auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .IsAtLocalSchemeShard(); + + if (streamPath.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeCdcStream, acceptExisted); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (checks) { - checks - .IsValidLeafName() - .PathsLimit() - .DirChildrenLimit(); - } - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - switch (streamDesc.GetMode()) { + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .PathsLimit() + .DirChildrenLimit(); + } + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + switch (streamDesc.GetMode()) { case NKikimrSchemeOp::ECdcStreamModeKeysOnly: case NKikimrSchemeOp::ECdcStreamModeUpdate: case NKikimrSchemeOp::ECdcStreamModeNewImage: case NKikimrSchemeOp::ECdcStreamModeOldImage: case NKikimrSchemeOp::ECdcStreamModeNewAndOldImages: - break; - default: + break; + default: return {CreateReject(opId, NKikimrScheme::StatusInvalidParameter, TStringBuilder() - << "Invalid stream mode: " << static_cast<ui32>(streamDesc.GetMode()))}; - } - - TString errStr; - if (!context.SS->CheckApplyIf(tx, errStr)) { + << "Invalid stream mode: " << static_cast<ui32>(streamDesc.GetMode()))}; + } + + TString errStr; + if (!context.SS->CheckApplyIf(tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusPreconditionFailed, errStr)}; - } - - if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { + } + + if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusMultipleModifications, errStr)}; - } - - TVector<ISubOperationBase::TPtr> result; - - { + } + + TVector<ISubOperationBase::TPtr> result; + + { auto outTx = TransactionTemplate(tablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStreamImpl); outTx.SetFailOnExist(!acceptExisted); - outTx.MutableCreateCdcStream()->CopyFrom(op); - - result.push_back(CreateNewCdcStreamImpl(NextPartId(opId, result), outTx)); - } - - { + outTx.MutableCreateCdcStream()->CopyFrom(op); + + result.push_back(CreateNewCdcStreamImpl(NextPartId(opId, result), outTx)); + } + + { auto outTx = TransactionTemplate(workingDirPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStreamAtTable); outTx.SetFailOnExist(!acceptExisted); - outTx.MutableCreateCdcStream()->CopyFrom(op); - - result.push_back(CreateNewCdcStreamAtTable(NextPartId(opId, result), outTx)); - } - - { - Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); - auto table = context.SS->Tables.at(tablePath.Base()->PathId); - const auto& partitions = table->GetPartitions(); - - auto outTx = TransactionTemplate(streamPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup); - outTx.SetFailOnExist(!acceptExisted); - - auto& desc = *outTx.MutableCreatePersQueueGroup(); - desc.SetName("streamImpl"); - desc.SetTotalGroupCount(partitions.size()); - desc.SetPartitionPerTablet(2); - - auto& pqConfig = *desc.MutablePQTabletConfig(); - auto& partitionConfig = *pqConfig.MutablePartitionConfig(); - partitionConfig.SetLifetimeSeconds(TDuration::Days(1).Seconds()); - - for (const auto& tag : table->KeyColumnIds) { - Y_VERIFY(table->Columns.contains(tag)); - const auto& column = table->Columns.at(tag); - - auto& keyComponent = *pqConfig.AddPartitionKeySchema(); - keyComponent.SetName(column.Name); - keyComponent.SetTypeId(column.PType); - } - - auto& bootstrapConfig = *desc.MutableBootstrapConfig(); - for (ui32 i = 0; i < partitions.size(); ++i) { - const auto& cur = partitions.at(i); - - Y_VERIFY(context.SS->ShardInfos.contains(cur.ShardIdx)); - const auto& shard = context.SS->ShardInfos.at(cur.ShardIdx); - - auto& mg = *bootstrapConfig.AddExplicitMessageGroups(); - mg.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(shard.TabletID))); - - if (i != partitions.size() - 1) { - TSerializedCellVec endKey(cur.EndOfRange); - Y_VERIFY(endKey.GetCells().size() <= table->KeyColumnIds.size()); - - TString errStr; - auto& boundary = *desc.AddPartitionBoundaries(); - for (ui32 ki = 0; ki < endKey.GetCells().size(); ++ki) { - const auto& cell = endKey.GetCells()[ki]; - const auto tag = table->KeyColumnIds.at(ki); - Y_VERIFY(table->Columns.contains(tag)); - const auto typeId = table->Columns.at(tag).PType; - const bool ok = NMiniKQL::CellToValue(typeId, cell, *boundary.AddTuple(), errStr); - Y_VERIFY(ok, "Failed to build key tuple at postition %" PRIu32 " error: %s", ki, errStr.data()); - } - - mg.MutableKeyRange()->SetToBound(cur.EndOfRange); - } - - if (i) { - const auto& prev = partitions.at(i - 1); - mg.MutableKeyRange()->SetFromBound(prev.EndOfRange); - } - } - - result.push_back(CreateNewPQ(NextPartId(opId, result), outTx)); - } - - return result; -} - + outTx.MutableCreateCdcStream()->CopyFrom(op); + + result.push_back(CreateNewCdcStreamAtTable(NextPartId(opId, result), outTx)); + } + + { + Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); + auto table = context.SS->Tables.at(tablePath.Base()->PathId); + const auto& partitions = table->GetPartitions(); + + auto outTx = TransactionTemplate(streamPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup); + outTx.SetFailOnExist(!acceptExisted); + + auto& desc = *outTx.MutableCreatePersQueueGroup(); + desc.SetName("streamImpl"); + desc.SetTotalGroupCount(partitions.size()); + desc.SetPartitionPerTablet(2); + + auto& pqConfig = *desc.MutablePQTabletConfig(); + auto& partitionConfig = *pqConfig.MutablePartitionConfig(); + partitionConfig.SetLifetimeSeconds(TDuration::Days(1).Seconds()); + + for (const auto& tag : table->KeyColumnIds) { + Y_VERIFY(table->Columns.contains(tag)); + const auto& column = table->Columns.at(tag); + + auto& keyComponent = *pqConfig.AddPartitionKeySchema(); + keyComponent.SetName(column.Name); + keyComponent.SetTypeId(column.PType); + } + + auto& bootstrapConfig = *desc.MutableBootstrapConfig(); + for (ui32 i = 0; i < partitions.size(); ++i) { + const auto& cur = partitions.at(i); + + Y_VERIFY(context.SS->ShardInfos.contains(cur.ShardIdx)); + const auto& shard = context.SS->ShardInfos.at(cur.ShardIdx); + + auto& mg = *bootstrapConfig.AddExplicitMessageGroups(); + mg.SetId(NPQ::NSourceIdEncoding::EncodeSimple(ToString(shard.TabletID))); + + if (i != partitions.size() - 1) { + TSerializedCellVec endKey(cur.EndOfRange); + Y_VERIFY(endKey.GetCells().size() <= table->KeyColumnIds.size()); + + TString errStr; + auto& boundary = *desc.AddPartitionBoundaries(); + for (ui32 ki = 0; ki < endKey.GetCells().size(); ++ki) { + const auto& cell = endKey.GetCells()[ki]; + const auto tag = table->KeyColumnIds.at(ki); + Y_VERIFY(table->Columns.contains(tag)); + const auto typeId = table->Columns.at(tag).PType; + const bool ok = NMiniKQL::CellToValue(typeId, cell, *boundary.AddTuple(), errStr); + Y_VERIFY(ok, "Failed to build key tuple at postition %" PRIu32 " error: %s", ki, errStr.data()); + } + + mg.MutableKeyRange()->SetToBound(cur.EndOfRange); + } + + if (i) { + const auto& prev = partitions.at(i - 1); + mg.MutableKeyRange()->SetFromBound(prev.EndOfRange); + } + } + + result.push_back(CreateNewPQ(NextPartId(opId, result), outTx)); + } + + return result; +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp index eac8d3fe4be..9b6b9a49600 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp @@ -11,8 +11,8 @@ using namespace NKikimr; using namespace NSchemeShard; class TCreateExtSubDomain: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -23,8 +23,8 @@ class TCreateExtSubDomain: public TSubOperation { switch(state) { case TTxState::Waiting: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -36,7 +36,7 @@ class TCreateExtSubDomain: public TSubOperation { case TTxState::Waiting: case TTxState::Propose: return THolder(new NSubDomainState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -53,11 +53,11 @@ class TCreateExtSubDomain: public TSubOperation { } public: - TCreateExtSubDomain(TOperationId id, const TTxTransaction& tx) + TCreateExtSubDomain(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateExtSubDomain(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -69,11 +69,11 @@ public: THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& settings = Transaction.GetSubDomain(); + const auto& settings = Transaction.GetSubDomain(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = settings.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = settings.GetName(); ui64 shardsToCreate = settings.GetCoordinators() + settings.GetMediators(); @@ -120,20 +120,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeExtSubDomain, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -144,8 +144,8 @@ public: .PathsLimit() //check capacity on root Domain .DirChildrenLimit() .PathShardsLimit(shardsToCreate) - .ShardsLimit(shardsToCreate) //check capacity on root Domain - .IsValidACL(acl); + .ShardsLimit(shardsToCreate) //check capacity on root Domain + .IsValidACL(acl); } if (!checks) { @@ -173,17 +173,17 @@ public: return result; } - TPathId resourcesDomainId; - if (settings.HasResourcesDomainKey()) { - const auto& resourcesDomainKey = settings.GetResourcesDomainKey(); - resourcesDomainId = TPathId(resourcesDomainKey.GetSchemeShard(), resourcesDomainKey.GetPathId()); - - if (!context.SS->SubDomains.contains(resourcesDomainId)) { + TPathId resourcesDomainId; + if (settings.HasResourcesDomainKey()) { + const auto& resourcesDomainKey = settings.GetResourcesDomainKey(); + resourcesDomainId = TPathId(resourcesDomainKey.GetSchemeShard(), resourcesDomainKey.GetPathId()); + + if (!context.SS->SubDomains.contains(resourcesDomainId)) { result->SetError(NKikimrScheme::StatusInvalidParameter, "Unknown resources domain key"); - return result; - } - } - + return result; + } + } + auto domainId = parentPath.DomainId(); Y_VERIFY(context.SS->PathsById.contains(domainId)); Y_VERIFY(context.SS->SubDomains.contains(domainId)); @@ -201,11 +201,11 @@ public: return result; } - const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); + const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); TUserAttributes::TPtr userAttrs = new TUserAttributes(1); - - TString errStr; - + + TString errStr; + if (!userAttrs->ApplyPatch(EUserAttributesOp::CreateExtSubDomain, userAttrsDetails, errStr) || !userAttrs->CheckLimits(errStr)) { @@ -213,7 +213,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -246,19 +246,19 @@ public: Y_VERIFY(!context.SS->FindTx(OperationId)); TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateExtSubDomain, newNode->PathId); - TSubDomainInfo::TPtr alter = new TSubDomainInfo(1, 0, 0, resourcesDomainId ? resourcesDomainId : newNode->PathId); + TSubDomainInfo::TPtr alter = new TSubDomainInfo(1, 0, 0, resourcesDomainId ? resourcesDomainId : newNode->PathId); alter->SetSchemeLimits(parentPath.DomainInfo()->GetSchemeLimits()); //inherit from root - if (resourcesDomainId) { - TSubDomainInfo::TPtr resourcesDomain = context.SS->SubDomains.at(resourcesDomainId); + if (resourcesDomainId) { + TSubDomainInfo::TPtr resourcesDomain = context.SS->SubDomains.at(resourcesDomainId); TTabletId sharedHive = context.SS->GetGlobalHive(context.Ctx); if (resourcesDomain->GetTenantHiveID()) { sharedHive = resourcesDomain->GetTenantHiveID(); - } + } alter->SetSharedHive(sharedHive); - } - + } + if (settings.HasDeclaredSchemeQuotas()) { alter->SetDeclaredSchemeQuotas(settings.GetDeclaredSchemeQuotas()); } @@ -303,16 +303,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateExtSubDomain"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateExtSubDomain"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateExtSubDomain AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -323,12 +323,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, const TTxTransaction& tx) { - return new TCreateExtSubDomain(id, tx); -} - +ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, const TTxTransaction& tx) { + return new TCreateExtSubDomain(id, tx); +} + ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateExtSubDomain(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index 1225d22343d..5beb0a88488 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -54,10 +54,10 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T } } - if (baseTableDescription.GetIsBackup()) { + if (baseTableDescription.GetIsBackup()) { return {CreateReject(nextId, NKikimrScheme::StatusInvalidParameter, "Cannot create table with explicit 'IsBackup' property")}; - } - + } + TSubDomainInfo::TPtr domainInfo = baseTablePath.DomainInfo(); if (sequencesCount > 0 && domainInfo->GetSequenceShards().empty()) { @@ -81,9 +81,9 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, msg)}; } - if (indexesCount > domainInfo->GetSchemeLimits().MaxTableIndices) { + if (indexesCount > domainInfo->GetSchemeLimits().MaxTableIndices) { auto msg = TStringBuilder() << "indexes count has reached maximum value in the table" - << ", children limit for dir in domain: " << domainInfo->GetSchemeLimits().MaxTableIndices + << ", children limit for dir in domain: " << domainInfo->GetSchemeLimits().MaxTableIndices << ", intention to create new children: " << indexesCount; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, msg)}; } @@ -128,28 +128,28 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; } - switch (indexDescription.GetType()) { + switch (indexDescription.GetType()) { case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: - if (!context.SS->EnableAsyncIndexes) { - TString msg = TStringBuilder() << "It is not allowed to create async indexes"; + if (!context.SS->EnableAsyncIndexes) { + TString msg = TStringBuilder() << "It is not allowed to create async indexes"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; - } else if (baseTableDescription.HasTTLSettings() && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { - TString msg = TStringBuilder() << "TTL is not currently supported on tables with async indices"; + } else if (baseTableDescription.HasTTLSettings() && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { + TString msg = TStringBuilder() << "TTL is not currently supported on tables with async indices"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; - } - break; - - default: - break; - } - + } + break; + + default: + break; + } + TIndexColumns indexKeys = ExtractInfo(indexDescription); if (indexKeys.KeyColumns.empty()) { TString msg = TStringBuilder() << "no key colums in index creation config"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; } - if (!indexKeys.DataColumns.empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { + if (!indexKeys.DataColumns.empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { TString msg = TStringBuilder() << "It is not allowed to create index with data column"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; } @@ -216,7 +216,7 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T } for (auto& column : baseTableDescription.GetColumns()) { - if (column.GetNotNull() && !AppData()->FeatureFlags.GetEnableNotNullColumns()) { + if (column.GetNotNull() && !AppData()->FeatureFlags.GetEnableNotNullColumns()) { TString msg = TStringBuilder() << "It is not allowed to create not null column"; return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; } @@ -254,17 +254,17 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T scheme.SetFailOnExist(tx.GetFailOnExist()); scheme.MutableCreateTable()->CopyFrom(baseTableDescription); - if (tx.HasAlterUserAttributes()) { + if (tx.HasAlterUserAttributes()) { scheme.MutableAlterUserAttributes()->CopyFrom(tx.GetAlterUserAttributes()); - } - + } + result.push_back(CreateNewTable(NextPartId(nextId, result), scheme, sequences)); } for (auto& indexDescription: indexedTable.GetIndexDescription()) { { auto scheme = TransactionTemplate( - tx.GetWorkingDir() + "/" + baseTableDescription.GetName(), + tx.GetWorkingDir() + "/" + baseTableDescription.GetName(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTableIndex); scheme.SetFailOnExist(tx.GetFailOnExist()); @@ -272,13 +272,13 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T if (!indexDescription.HasType()) { scheme.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal); } - + result.push_back(CreateNewTableIndex(NextPartId(nextId, result), scheme)); } { auto scheme = TransactionTemplate( - tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + indexDescription.GetName(), + tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + indexDescription.GetName(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); scheme.SetFailOnExist(tx.GetFailOnExist()); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_kesus.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_kesus.cpp index e47fa4a7bd7..c974b72c318 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_kesus.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_kesus.cpp @@ -227,9 +227,9 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -251,8 +251,8 @@ public: class TCreateKesus: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -266,8 +266,8 @@ class TCreateKesus: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -283,7 +283,7 @@ class TCreateKesus: public TSubOperation { return THolder(new TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -300,11 +300,11 @@ class TCreateKesus: public TSubOperation { } public: - TCreateKesus(TOperationId id, const TTxTransaction& tx) + TCreateKesus(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateKesus(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -317,10 +317,10 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const auto& config = Transaction.GetKesus().GetConfig(); + const auto& config = Transaction.GetKesus().GetConfig(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetKesus().GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetKesus().GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateKesus Propose" @@ -331,7 +331,7 @@ public: TEvSchemeShard::EStatus status = NKikimrScheme::StatusAccepted; auto result = MakeHolder<TProposeResponse>(status, ui64(OperationId.GetTxId()), ui64(ssId)); - TString errStr; + TString errStr; if (!ValidateConfig(config, status, errStr)) { result->SetError(status, errStr); return result; @@ -358,20 +358,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeKesus, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -382,8 +382,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit() - .PathShardsLimit() - .IsValidACL(acl); + .PathShardsLimit() + .IsValidACL(acl); } if (!checks) { @@ -407,7 +407,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -442,16 +442,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateKesus"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateKesus"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateKesus AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -462,12 +462,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateNewKesus(TOperationId id, const TTxTransaction& tx) { - return new TCreateKesus(id, tx); -} - +ISubOperationBase::TPtr CreateNewKesus(TOperationId id, const TTxTransaction& tx) { + return new TCreateKesus(id, tx); +} + ISubOperationBase::TPtr CreateNewKesus(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateKesus(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index 7654bf7d814..16ff55eaa66 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -187,12 +187,12 @@ public: result->SetPathId(pathId.LocalPathId); TTableInfo::TPtr tableInfo = context.SS->Tables.at(tablePath.Base()->PathId); - - if (tableInfo->IsTTLEnabled()) { - if (indexSchema.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { + + if (tableInfo->IsTTLEnabled()) { + if (indexSchema.GetType() == NKikimrSchemeOp::EIndexTypeGlobalAsync && !AppData()->FeatureFlags.GetEnableTtlOnAsyncIndexedTables()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "Async indexes are not currently supported on tables with TTL"); - return result; - } + return result; + } } if (tablePath.Base()->GetAliveChildren() + 1 > tablePath.DomainInfo()->GetSchemeLimits().MaxTableIndices) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp index 37bce50203b..a3e6896b6f1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp @@ -670,7 +670,7 @@ public: return result; } - if (!AppData()->FeatureFlags.GetEnableOlapSchemaOperations()) { + if (!AppData()->FeatureFlags.GetEnableOlapSchemaOperations()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "Olap schema operations are not supported"); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp index 111327dc3b7..09f556c6558 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp @@ -15,7 +15,7 @@ namespace { TOlapTableInfo::TPtr CreateOlapTable( const NKikimrSchemeOp::TColumnTableDescription& opSrc, - TOlapStoreInfo::TPtr storeInfo, const TSubDomainInfo& subDomain, + TOlapStoreInfo::TPtr storeInfo, const TSubDomainInfo& subDomain, TEvSchemeShard::EStatus& status, TString& errStr, TSchemeShard* ss) { @@ -760,13 +760,13 @@ public: } } - if (!AppData()->FeatureFlags.GetEnableOlapSchemaOperations()) { + if (!AppData()->FeatureFlags.GetEnableOlapSchemaOperations()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "Olap schema operations are not supported"); return result; } - TOlapTableInfo::TPtr tableInfo = CreateOlapTable(createDescription, storeInfo, *parentPath.DomainInfo(), status, errStr, context.SS); + TOlapTableInfo::TPtr tableInfo = CreateOlapTable(createDescription, storeInfo, *parentPath.DomainInfo(), status, errStr, context.SS); if (!tableInfo) { result->SetError(status, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp index 3a3d5037590..992fd3175c2 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp @@ -45,12 +45,12 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, return nullptr; } - if (!op.HasPQTabletConfig()) { + if (!op.HasPQTabletConfig()) { status = NKikimrScheme::StatusSchemeError; - errStr = Sprintf("No PQTabletConfig specified"); - return nullptr; - } - + errStr = Sprintf("No PQTabletConfig specified"); + return nullptr; + } + if ((ui32)op.GetPQTabletConfig().GetPartitionConfig().GetWriteSpeedInBytesPerSecond() > TSchemeShard::MaxPQWriteSpeedPerPartition) { status = NKikimrScheme::StatusInvalidParameter; errStr = TStringBuilder() @@ -67,56 +67,56 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, return nullptr; } - if (op.GetPQTabletConfig().PartitionKeySchemaSize()) { - if (op.PartitionBoundariesSize() != (partitionCount - 1)) { + if (op.GetPQTabletConfig().PartitionKeySchemaSize()) { + if (op.PartitionBoundariesSize() != (partitionCount - 1)) { status = NKikimrScheme::StatusInvalidParameter; - errStr = Sprintf("Partition count and partition boundaries size mismatch: %lu, %u", - op.PartitionBoundariesSize(), partitionCount); - return nullptr; - } - - TString error; - if (!pqGroupInfo->FillKeySchema(op.GetPQTabletConfig(), error)) { + errStr = Sprintf("Partition count and partition boundaries size mismatch: %lu, %u", + op.PartitionBoundariesSize(), partitionCount); + return nullptr; + } + + TString error; + if (!pqGroupInfo->FillKeySchema(op.GetPQTabletConfig(), error)) { status = NKikimrScheme::StatusSchemeError; - errStr = Sprintf("Invalid key schema: %s", error.data()); - return nullptr; - } - } else { - if (op.PartitionBoundariesSize()) { + errStr = Sprintf("Invalid key schema: %s", error.data()); + return nullptr; + } + } else { + if (op.PartitionBoundariesSize()) { status = NKikimrScheme::StatusInvalidParameter; - errStr = "Missing key schema with specified partition boundaries"; - return nullptr; - } - } - - TString prevBound; + errStr = "Missing key schema with specified partition boundaries"; + return nullptr; + } + } + + TString prevBound; for (ui32 i = 0; i < partitionCount; ++i) { - using TKeyRange = TPQShardInfo::TKeyRange; - TMaybe<TKeyRange> keyRange; - - if (op.PartitionBoundariesSize()) { - keyRange.ConstructInPlace(); - - if (i) { - keyRange->FromBound = prevBound; - } - - if (i != (partitionCount - 1)) { - TVector<TCell> cells; - TString error; - if (!NMiniKQL::CellsFromTuple(nullptr, op.GetPartitionBoundaries(i), pqGroupInfo->KeySchema, false, cells, error)) { + using TKeyRange = TPQShardInfo::TKeyRange; + TMaybe<TKeyRange> keyRange; + + if (op.PartitionBoundariesSize()) { + keyRange.ConstructInPlace(); + + if (i) { + keyRange->FromBound = prevBound; + } + + if (i != (partitionCount - 1)) { + TVector<TCell> cells; + TString error; + if (!NMiniKQL::CellsFromTuple(nullptr, op.GetPartitionBoundaries(i), pqGroupInfo->KeySchema, false, cells, error)) { status = NKikimrScheme::StatusSchemeError; - errStr = Sprintf("Invalid partition boundary at position: %u, error: %s", i, error.data()); - return nullptr; - } - - cells.resize(pqGroupInfo->KeySchema.size()); // Extend with NULLs - keyRange->ToBound = TSerializedCellVec::Serialize(cells); - prevBound = *keyRange->ToBound; - } - } - - pqGroupInfo->PartitionsToAdd.emplace(i, i + 1, keyRange); + errStr = Sprintf("Invalid partition boundary at position: %u, error: %s", i, error.data()); + return nullptr; + } + + cells.resize(pqGroupInfo->KeySchema.size()); // Extend with NULLs + keyRange->ToBound = TSerializedCellVec::Serialize(cells); + prevBound = *keyRange->ToBound; + } + } + + pqGroupInfo->PartitionsToAdd.emplace(i, i + 1, keyRange); } if (partsPerTablet == 0 || partsPerTablet > TSchemeShard::MaxPQTabletPartitionsCount) { @@ -139,7 +139,7 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, } NKikimrPQ::TPQTabletConfig tabletConfig = op.GetPQTabletConfig(); - tabletConfig.ClearPartitionIds(); + tabletConfig.ClearPartitionIds(); tabletConfig.ClearPartitions(); if (!CheckPersQueueConfig(tabletConfig, false, &errStr)) { @@ -165,10 +165,10 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, Y_PROTOBUF_SUPPRESS_NODISCARD tabletConfig.SerializeToString(&pqGroupInfo->TabletConfig); - if (op.HasBootstrapConfig()) { - Y_PROTOBUF_SUPPRESS_NODISCARD op.GetBootstrapConfig().SerializeToString(&pqGroupInfo->BootstrapConfig); - } - + if (op.HasBootstrapConfig()) { + Y_PROTOBUF_SUPPRESS_NODISCARD op.GetBootstrapConfig().SerializeToString(&pqGroupInfo->BootstrapConfig); + } + return pqGroupInfo; } @@ -209,9 +209,9 @@ void ApplySharding(TTxId txId, TPQShardInfo::TPtr pqShard = pqGroup->Shards[idx]; TPQShardInfo::TPersQueueInfo pqInfo; - pqInfo.PqId = it->PartitionId; - pqInfo.GroupId = it->GroupId; - pqInfo.KeyRange = it->KeyRange; + pqInfo.PqId = it->PartitionId; + pqInfo.GroupId = it->GroupId; + pqInfo.KeyRange = it->KeyRange; pqInfo.AlterVersion = 1; pqShard->PQInfos.push_back(pqInfo); } @@ -219,8 +219,8 @@ void ApplySharding(TTxId txId, class TCreatePQ: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -234,8 +234,8 @@ class TCreatePQ: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -251,7 +251,7 @@ class TCreatePQ: public TSubOperation { return THolder(new NPQState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NPQState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -268,11 +268,11 @@ class TCreatePQ: public TSubOperation { } public: - TCreatePQ(TOperationId id, const TTxTransaction& tx) + TCreatePQ(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreatePQ(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -285,14 +285,14 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const auto& createDEscription = Transaction.GetCreatePersQueueGroup(); + const auto& createDEscription = Transaction.GetCreatePersQueueGroup(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = createDEscription.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreatePQ Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", opId: " << OperationId << ", at schemeshard: " << ssId); @@ -307,20 +307,20 @@ public: .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() - .NotUnderDeleting(); - - if (checks) { - if (parentPath.Base()->IsCdcStream()) { - checks - .IsUnderCreating(NKikimrScheme::StatusNameConflict) - .IsUnderTheSameOperation(OperationId.GetTxId()); - } else { - checks - .IsCommonSensePath() - .IsLikeDirectory(); - } - } - + .NotUnderDeleting(); + + if (checks) { + if (parentPath.Base()->IsCdcStream()) { + checks + .IsUnderCreating(NKikimrScheme::StatusNameConflict) + .IsUnderTheSameOperation(OperationId.GetTxId()); + } else { + checks + .IsCommonSensePath() + .IsLikeDirectory(); + } + } + if (!checks) { TString explain = TStringBuilder() << "parent path fail checks" << ", path: " << parentPath.PathString(); @@ -330,20 +330,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypePersQueueGroup, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -352,8 +352,8 @@ public: .IsValidLeafName() .DepthLimit() .PathsLimit() - .DirChildrenLimit() - .IsValidACL(acl); + .DirChildrenLimit() + .IsValidACL(acl); } if (!checks) { @@ -369,9 +369,9 @@ public: } } - TString errStr; - - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -481,7 +481,7 @@ public: for (auto& shard : pqGroup->Shards) { auto shardIdx = shard.first; for (const auto& pqInfo : shard.second->PQInfos) { - context.SS->PersistPersQueue(db, pathId, shardIdx, pqInfo); + context.SS->PersistPersQueue(db, pathId, shardIdx, pqInfo); } } @@ -520,7 +520,7 @@ public: context.SS->PersistTxState(db, OperationId); context.SS->PersistPath(db, dstPath.Base()->PathId); - + if (!acl.empty()) { dstPath.Base()->ApplyACL(acl); context.SS->PersistACL(db, dstPath.Base()); @@ -555,16 +555,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreatePQ"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreatePQ"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreatePQ AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -575,12 +575,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateNewPQ(TOperationId id, const TTxTransaction& tx) { - return new TCreatePQ(id, tx); -} - +ISubOperationBase::TPtr CreateNewPQ(TOperationId id, const TTxTransaction& tx) { + return new TCreatePQ(id, tx); +} + ISubOperationBase::TPtr CreateNewPQ(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreatePQ(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp index 03a5b220b6a..c6458f7aa59 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp @@ -1,476 +1,476 @@ -#include "schemeshard__operation_part.h" -#include "schemeshard__operation_common.h" -#include "schemeshard_impl.h" - -#include <ydb/core/tx/replication/controller/public_events.h> - -#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_W(stream) LOG_WARN_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) - -namespace NKikimr { -namespace NSchemeShard { - -namespace { - -class TConfigureParts: public TSubOperationState { - TString DebugHint() const override { - return TStringBuilder() - << "TCreateReplication TConfigureParts" - << " opId# " << OperationId << " "; - } - -public: - explicit TConfigureParts(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), { - TEvHive::TEvCreateTabletReply::EventType, - }); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->Replications.contains(pathId)); - auto alterData = context.SS->Replications.at(pathId)->AlterData; - Y_VERIFY(alterData); - - txState->ClearShardsInProgress(); - - for (const auto& shard : txState->Shards) { - Y_VERIFY(shard.TabletType == ETabletType::ReplicationController); - - Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); - const auto tabletId = context.SS->ShardInfos.at(shard.Idx).TabletID; - - if (tabletId == InvalidTabletId) { - LOG_D(DebugHint() << "Shard is not created yet" - << ": shardIdx# " << shard.Idx); - context.OnComplete.WaitShardCreated(shard.Idx, OperationId); - } else { - auto ev = MakeHolder<NReplication::TEvController::TEvCreateReplication>(); - PathIdFromPathId(pathId, ev->Record.MutablePathId()); - ev->Record.MutableOperationId()->SetTxId(ui64(OperationId.GetTxId())); - ev->Record.MutableOperationId()->SetPartId(ui32(OperationId.GetSubTxId())); - ev->Record.MutableConfig()->CopyFrom(alterData->Description.GetConfig()); - - LOG_D(DebugHint() << "Send TEvCreateReplication to controller" - << ": tabletId# " << tabletId - << ", ev# " << ev->ToString()); - context.OnComplete.BindMsgToPipe(OperationId, tabletId, pathId, ev.Release()); - } - - txState->ShardsInProgress.insert(shard.Idx); - } - - return false; - } - - bool HandleReply(NReplication::TEvController::TEvCreateReplicationResult::TPtr& ev, TOperationContext& context) override { - LOG_I(DebugHint() << "HandleReply " << ev->Get()->ToString()); - - const auto tabletId = TTabletId(ev->Get()->Record.GetOrigin()); - const auto status = ev->Get()->Record.GetStatus(); - - switch (status) { - case NKikimrReplication::TEvCreateReplicationResult::SUCCESS: - case NKikimrReplication::TEvCreateReplicationResult::ALREADY_EXISTS: - break; - default: - LOG_W(DebugHint() << "Ignoring unexpected TEvCreateReplicationResult" - << " tabletId# " << tabletId - << " status# " << static_cast<int>(status)); - return false; - } - - auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); - Y_VERIFY(txState->State == TTxState::ConfigureParts); - - const auto shardIdx = context.SS->MustGetShardIdx(tabletId); - if (!txState->ShardsInProgress.erase(shardIdx)) { - LOG_W(DebugHint() << "Ignoring duplicate TEvCreateReplicationResult"); - return false; - } - - context.OnComplete.UnbindMsgFromPipe(OperationId, tabletId, txState->TargetPathId); - - if (!txState->ShardsInProgress.empty()) { - return false; - } - - NIceDb::TNiceDb db(context.Txc.DB); - context.SS->ChangeTxState(db, OperationId, TTxState::Propose); - context.OnComplete.ActivateTx(OperationId); - - return true; - } - -private: - const TOperationId OperationId; - -}; // TConfigureParts - -class TPropose: public TSubOperationState { - TString DebugHint() const override { - return TStringBuilder() - << "TCreateReplication TPropose" - << " opId# " << OperationId << " "; - } - -public: - explicit TPropose(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), { - TEvHive::TEvCreateTabletReply::EventType, - NReplication::TEvController::TEvCreateReplicationResult::EventType, - }); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - const auto step = TStepId(ev->Get()->StepId); - - LOG_I(DebugHint() << "HandleReply TEvOperationPlan" - << ": step# " << step); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Replications.contains(pathId)); - auto replication = context.SS->Replications.at(pathId); - - auto alterData = replication->AlterData; - Y_VERIFY(alterData); - - NIceDb::TNiceDb db(context.Txc.DB); - - path->StepCreated = step; - context.SS->PersistCreateStep(db, pathId, step); - - context.SS->Replications[pathId] = alterData; - context.SS->PersistReplicationAlterRemove(db, pathId); - context.SS->PersistReplication(db, pathId, *alterData); - - Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); - auto parentPath = context.SS->PathsById.at(path->ParentPathId); - - ++parentPath->DirAlterVersion; - context.SS->PersistPathDirAlterVersion(db, parentPath); - - context.SS->ClearDescribePathCaches(parentPath); - context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); - - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - return true; - } - -private: - const TOperationId OperationId; - -}; // TPropose - -class TCreateReplication: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::CreateParts; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::CreateParts: - return TTxState::ConfigureParts; - case TTxState::ConfigureParts: - return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; - default: - return TTxState::Invalid; - } - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::CreateParts: - return MakeHolder<TCreateParts>(OperationId); - case TTxState::ConfigureParts: - return MakeHolder<TConfigureParts>(OperationId); - case TTxState::Propose: - return MakeHolder<TPropose>(OperationId); - case TTxState::Done: - return MakeHolder<TDone>(OperationId); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TCreateReplication(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TCreateReplication(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - - THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - auto desc = Transaction.GetReplication(); - const auto& name = desc.GetName(); - const auto& acl = Transaction.GetModifyACL().GetDiffACL(); - const auto acceptExisted = !Transaction.GetFailOnExist(); - - LOG_N("TCreateReplication Propose" - << ": opId# " << OperationId - << ", path# " << workingDir << "/" << name); - - auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(context.SS->SelfTabletId())); - - const auto parentPath = TPath::Resolve(workingDir, context.SS); - { - const auto checks = parentPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() - .IsLikeDirectory(); - - if (!checks) { - TString explain = TStringBuilder() << "parent path checks failed, path: " << parentPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - - return result; - } - } - - const auto domainId = parentPath.DomainId(); - auto domainInfo = parentPath.DomainInfo(); - const ui64 shardsToCreate = domainInfo->GetReplicationControllers().empty(); - - auto path = parentPath.Child(name); - { - const auto checks = path.Check(); - checks - .IsAtLocalSchemeShard(); - - if (path.IsResolved()) { - checks - .IsResolved() - .NotUnderDeleting() - .FailOnExist(TPathElement::EPathType::EPathTypeReplication, acceptExisted); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (checks) { - checks - .IsValidLeafName() - .DepthLimit() - .PathsLimit() - .DirChildrenLimit() - .ShardsLimit(shardsToCreate) - .IsValidACL(acl); - } - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << path.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - if (path.IsResolved()) { - result->SetPathCreateTxId(ui64(path->CreateTxId)); - result->SetPathId(path->PathId.LocalPathId); - } - - return result; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { - result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); - return result; - } - - TChannelsBindings channelsBindings; - if (shardsToCreate) { - if (!context.SS->ResolveTabletChannels(0, domainId, channelsBindings)) { - result->SetError(NKikimrScheme::StatusInvalidParameter, - "Unable to construct channel binding for replication controller with the storage pool"); - return result; - } - } - - path.MaterializeLeaf(owner); - path->CreateTxId = OperationId.GetTxId(); - path->LastTxId = OperationId.GetTxId(); - path->PathState = TPathElement::EPathState::EPathStateCreate; - path->PathType = TPathElement::EPathType::EPathTypeReplication; - result->SetPathId(path->PathId.LocalPathId); - - context.SS->IncrementPathDbRefCount(path->PathId); - parentPath->IncAliveChildren(); - domainInfo->IncPathsInside(); - context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateReplication, path->PathId); - - if (shardsToCreate) { - const auto shardIdx = context.SS->RegisterShardInfo( - TShardInfo::ReplicationControllerInfo(OperationId.GetTxId(), domainId) - .WithBindedChannels(channelsBindings)); - context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Add(1); - - txState.Shards.emplace_back(shardIdx, ETabletType::ReplicationController, TTxState::CreateParts); - txState.State = TTxState::CreateParts; - - Y_VERIFY(context.SS->PathsById.contains(domainId)); - context.SS->PathsById.at(domainId)->IncShardsInside(); - - domainInfo->AddInternalShard(shardIdx); - domainInfo->AddReplicationController(shardIdx); - } else { - const auto shardIdx = *domainInfo->GetReplicationControllers().begin(); - - txState.Shards.emplace_back(shardIdx, ETabletType::ReplicationController, TTxState::ConfigureParts); - txState.State = TTxState::ConfigureParts; - - Y_VERIFY(context.SS->ShardInfos.contains(shardIdx)); - const auto& shardInfo = context.SS->ShardInfos.at(shardIdx); - - if (shardInfo.CurrentTxId != OperationId.GetTxId()) { - context.OnComplete.Dependence(shardInfo.CurrentTxId, OperationId.GetTxId()); - } - } - - if (parentPath->HasActiveChanges()) { - const auto parentTxId = parentPath->PlannedToCreate() ? parentPath->CreateTxId : parentPath->LastTxId; - context.OnComplete.Dependence(parentTxId, OperationId.GetTxId()); - } - - NIceDb::TNiceDb db(context.Txc.DB); - - context.SS->PersistPath(db, path->PathId); - if (!acl.empty()) { - path->ApplyACL(acl); - context.SS->PersistACL(db, path.Base()); - } - - auto replication = TReplicationInfo::Create(std::move(desc)); - context.SS->Replications[path->PathId] = replication; - context.SS->PersistReplication(db, path->PathId, *replication); - context.SS->PersistReplicationAlter(db, path->PathId, *replication->AlterData); - - Y_VERIFY(txState.Shards.size() == 1); - for (const auto& shard : txState.Shards) { - Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); - const TShardInfo& shardInfo = context.SS->ShardInfos.at(shard.Idx); - - if (shard.Operation == TTxState::CreateParts) { - context.SS->PersistShardMapping(db, shard.Idx, InvalidTabletId, domainId, OperationId.GetTxId(), shard.TabletType); - context.SS->PersistChannelsBinding(db, shard.Idx, shardInfo.BindedChannels); - } - } - - context.SS->ChangeTxState(db, OperationId, txState.State); - context.SS->PersistTxState(db, OperationId); - context.SS->PersistUpdateNextPathId(db); - context.SS->PersistUpdateNextShardIdx(db); - - ++parentPath->DirAlterVersion; - context.SS->PersistPathDirAlterVersion(db, parentPath.Base()); - - context.SS->ClearDescribePathCaches(parentPath.Base()); - context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); - - context.SS->ClearDescribePathCaches(path.Base()); - context.OnComplete.PublishToSchemeBoard(OperationId, path->PathId); - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateReplication"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TCreateReplication AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TCreateReplication - -} // anonymous - -ISubOperationBase::TPtr CreateNewReplication(TOperationId id, const TTxTransaction& tx) { - return new TCreateReplication(id, tx); -} - -ISubOperationBase::TPtr CreateNewReplication(TOperationId id, TTxState::ETxState state) { - return new TCreateReplication(id, state); -} - -} // NSchemeShard -} // NKikimr +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#include <ydb/core/tx/replication/controller/public_events.h> + +#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_W(stream) LOG_WARN_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr { +namespace NSchemeShard { + +namespace { + +class TConfigureParts: public TSubOperationState { + TString DebugHint() const override { + return TStringBuilder() + << "TCreateReplication TConfigureParts" + << " opId# " << OperationId << " "; + } + +public: + explicit TConfigureParts(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), { + TEvHive::TEvCreateTabletReply::EventType, + }); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->Replications.contains(pathId)); + auto alterData = context.SS->Replications.at(pathId)->AlterData; + Y_VERIFY(alterData); + + txState->ClearShardsInProgress(); + + for (const auto& shard : txState->Shards) { + Y_VERIFY(shard.TabletType == ETabletType::ReplicationController); + + Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); + const auto tabletId = context.SS->ShardInfos.at(shard.Idx).TabletID; + + if (tabletId == InvalidTabletId) { + LOG_D(DebugHint() << "Shard is not created yet" + << ": shardIdx# " << shard.Idx); + context.OnComplete.WaitShardCreated(shard.Idx, OperationId); + } else { + auto ev = MakeHolder<NReplication::TEvController::TEvCreateReplication>(); + PathIdFromPathId(pathId, ev->Record.MutablePathId()); + ev->Record.MutableOperationId()->SetTxId(ui64(OperationId.GetTxId())); + ev->Record.MutableOperationId()->SetPartId(ui32(OperationId.GetSubTxId())); + ev->Record.MutableConfig()->CopyFrom(alterData->Description.GetConfig()); + + LOG_D(DebugHint() << "Send TEvCreateReplication to controller" + << ": tabletId# " << tabletId + << ", ev# " << ev->ToString()); + context.OnComplete.BindMsgToPipe(OperationId, tabletId, pathId, ev.Release()); + } + + txState->ShardsInProgress.insert(shard.Idx); + } + + return false; + } + + bool HandleReply(NReplication::TEvController::TEvCreateReplicationResult::TPtr& ev, TOperationContext& context) override { + LOG_I(DebugHint() << "HandleReply " << ev->Get()->ToString()); + + const auto tabletId = TTabletId(ev->Get()->Record.GetOrigin()); + const auto status = ev->Get()->Record.GetStatus(); + + switch (status) { + case NKikimrReplication::TEvCreateReplicationResult::SUCCESS: + case NKikimrReplication::TEvCreateReplicationResult::ALREADY_EXISTS: + break; + default: + LOG_W(DebugHint() << "Ignoring unexpected TEvCreateReplicationResult" + << " tabletId# " << tabletId + << " status# " << static_cast<int>(status)); + return false; + } + + auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); + Y_VERIFY(txState->State == TTxState::ConfigureParts); + + const auto shardIdx = context.SS->MustGetShardIdx(tabletId); + if (!txState->ShardsInProgress.erase(shardIdx)) { + LOG_W(DebugHint() << "Ignoring duplicate TEvCreateReplicationResult"); + return false; + } + + context.OnComplete.UnbindMsgFromPipe(OperationId, tabletId, txState->TargetPathId); + + if (!txState->ShardsInProgress.empty()) { + return false; + } + + NIceDb::TNiceDb db(context.Txc.DB); + context.SS->ChangeTxState(db, OperationId, TTxState::Propose); + context.OnComplete.ActivateTx(OperationId); + + return true; + } + +private: + const TOperationId OperationId; + +}; // TConfigureParts + +class TPropose: public TSubOperationState { + TString DebugHint() const override { + return TStringBuilder() + << "TCreateReplication TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), { + TEvHive::TEvCreateTabletReply::EventType, + NReplication::TEvController::TEvCreateReplicationResult::EventType, + }); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateReplication); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Replications.contains(pathId)); + auto replication = context.SS->Replications.at(pathId); + + auto alterData = replication->AlterData; + Y_VERIFY(alterData); + + NIceDb::TNiceDb db(context.Txc.DB); + + path->StepCreated = step; + context.SS->PersistCreateStep(db, pathId, step); + + context.SS->Replications[pathId] = alterData; + context.SS->PersistReplicationAlterRemove(db, pathId); + context.SS->PersistReplication(db, pathId, *alterData); + + Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); + auto parentPath = context.SS->PathsById.at(path->ParentPathId); + + ++parentPath->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentPath); + + context.SS->ClearDescribePathCaches(parentPath); + context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); + + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TCreateReplication: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::CreateParts; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::CreateParts: + return TTxState::ConfigureParts; + case TTxState::ConfigureParts: + return TTxState::Propose; + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::CreateParts: + return MakeHolder<TCreateParts>(OperationId); + case TTxState::ConfigureParts: + return MakeHolder<TConfigureParts>(OperationId); + case TTxState::Propose: + return MakeHolder<TPropose>(OperationId); + case TTxState::Done: + return MakeHolder<TDone>(OperationId); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TCreateReplication(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TCreateReplication(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + + THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { + const auto& workingDir = Transaction.GetWorkingDir(); + auto desc = Transaction.GetReplication(); + const auto& name = desc.GetName(); + const auto& acl = Transaction.GetModifyACL().GetDiffACL(); + const auto acceptExisted = !Transaction.GetFailOnExist(); + + LOG_N("TCreateReplication Propose" + << ": opId# " << OperationId + << ", path# " << workingDir << "/" << name); + + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(context.SS->SelfTabletId())); + + const auto parentPath = TPath::Resolve(workingDir, context.SS); + { + const auto checks = parentPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory(); + + if (!checks) { + TString explain = TStringBuilder() << "parent path checks failed, path: " << parentPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + + return result; + } + } + + const auto domainId = parentPath.DomainId(); + auto domainInfo = parentPath.DomainInfo(); + const ui64 shardsToCreate = domainInfo->GetReplicationControllers().empty(); + + auto path = parentPath.Child(name); + { + const auto checks = path.Check(); + checks + .IsAtLocalSchemeShard(); + + if (path.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeReplication, acceptExisted); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .DepthLimit() + .PathsLimit() + .DirChildrenLimit() + .ShardsLimit(shardsToCreate) + .IsValidACL(acl); + } + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << path.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + if (path.IsResolved()) { + result->SetPathCreateTxId(ui64(path->CreateTxId)); + result->SetPathId(path->PathId.LocalPathId); + } + + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + TChannelsBindings channelsBindings; + if (shardsToCreate) { + if (!context.SS->ResolveTabletChannels(0, domainId, channelsBindings)) { + result->SetError(NKikimrScheme::StatusInvalidParameter, + "Unable to construct channel binding for replication controller with the storage pool"); + return result; + } + } + + path.MaterializeLeaf(owner); + path->CreateTxId = OperationId.GetTxId(); + path->LastTxId = OperationId.GetTxId(); + path->PathState = TPathElement::EPathState::EPathStateCreate; + path->PathType = TPathElement::EPathType::EPathTypeReplication; + result->SetPathId(path->PathId.LocalPathId); + + context.SS->IncrementPathDbRefCount(path->PathId); + parentPath->IncAliveChildren(); + domainInfo->IncPathsInside(); + context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateReplication, path->PathId); + + if (shardsToCreate) { + const auto shardIdx = context.SS->RegisterShardInfo( + TShardInfo::ReplicationControllerInfo(OperationId.GetTxId(), domainId) + .WithBindedChannels(channelsBindings)); + context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_CONTROLLER_COUNT].Add(1); + + txState.Shards.emplace_back(shardIdx, ETabletType::ReplicationController, TTxState::CreateParts); + txState.State = TTxState::CreateParts; + + Y_VERIFY(context.SS->PathsById.contains(domainId)); + context.SS->PathsById.at(domainId)->IncShardsInside(); + + domainInfo->AddInternalShard(shardIdx); + domainInfo->AddReplicationController(shardIdx); + } else { + const auto shardIdx = *domainInfo->GetReplicationControllers().begin(); + + txState.Shards.emplace_back(shardIdx, ETabletType::ReplicationController, TTxState::ConfigureParts); + txState.State = TTxState::ConfigureParts; + + Y_VERIFY(context.SS->ShardInfos.contains(shardIdx)); + const auto& shardInfo = context.SS->ShardInfos.at(shardIdx); + + if (shardInfo.CurrentTxId != OperationId.GetTxId()) { + context.OnComplete.Dependence(shardInfo.CurrentTxId, OperationId.GetTxId()); + } + } + + if (parentPath->HasActiveChanges()) { + const auto parentTxId = parentPath->PlannedToCreate() ? parentPath->CreateTxId : parentPath->LastTxId; + context.OnComplete.Dependence(parentTxId, OperationId.GetTxId()); + } + + NIceDb::TNiceDb db(context.Txc.DB); + + context.SS->PersistPath(db, path->PathId); + if (!acl.empty()) { + path->ApplyACL(acl); + context.SS->PersistACL(db, path.Base()); + } + + auto replication = TReplicationInfo::Create(std::move(desc)); + context.SS->Replications[path->PathId] = replication; + context.SS->PersistReplication(db, path->PathId, *replication); + context.SS->PersistReplicationAlter(db, path->PathId, *replication->AlterData); + + Y_VERIFY(txState.Shards.size() == 1); + for (const auto& shard : txState.Shards) { + Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); + const TShardInfo& shardInfo = context.SS->ShardInfos.at(shard.Idx); + + if (shard.Operation == TTxState::CreateParts) { + context.SS->PersistShardMapping(db, shard.Idx, InvalidTabletId, domainId, OperationId.GetTxId(), shard.TabletType); + context.SS->PersistChannelsBinding(db, shard.Idx, shardInfo.BindedChannels); + } + } + + context.SS->ChangeTxState(db, OperationId, txState.State); + context.SS->PersistTxState(db, OperationId); + context.SS->PersistUpdateNextPathId(db); + context.SS->PersistUpdateNextShardIdx(db); + + ++parentPath->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentPath.Base()); + + context.SS->ClearDescribePathCaches(parentPath.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); + + context.SS->ClearDescribePathCaches(path.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, path->PathId); + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateReplication"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TCreateReplication AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TCreateReplication + +} // anonymous + +ISubOperationBase::TPtr CreateNewReplication(TOperationId id, const TTxTransaction& tx) { + return new TCreateReplication(id, tx); +} + +ISubOperationBase::TPtr CreateNewReplication(TOperationId id, TTxState::ETxState state) { + return new TCreateReplication(id, state); +} + +} // NSchemeShard +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_restore.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_restore.cpp index eb75cf06ecd..44b7a59e702 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_restore.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_restore.cpp @@ -1,128 +1,128 @@ -#include "schemeshard__operation_backup_restore_common.h" -#include "schemeshard_billing_helpers.h" - -namespace NKikimr { +#include "schemeshard__operation_backup_restore_common.h" +#include "schemeshard_billing_helpers.h" + +namespace NKikimr { namespace NSchemeShard { - -struct TRestore { - static constexpr TStringBuf Name() { - return "TRestore"; - } - - static bool HasTask(const TTxTransaction& tx) { - return tx.HasRestore(); - } - - static TString GetTableName(const TTxTransaction& tx) { - return tx.GetRestore().GetTableName(); - } - - static void ProposeTx(const TOperationId& opId, TTxState& txState, TOperationContext& context) { - const auto& pathId = txState.TargetPathId; - - const auto seqNo = context.SS->StartRound(txState); - const auto& processingParams = context.SS->SelectProcessingPrarams(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - TTableInfo::TPtr table = context.SS->Tables.at(pathId); - - for (ui32 i = 0; i < txState.Shards.size(); ++i) { - const auto& idx = txState.Shards[i].Idx; - const auto& datashardId = context.SS->ShardInfos[idx].TabletID; - - LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Propose restore" - << ", datashard: " << datashardId - << ", opId: " << opId - << ", at schemeshard: " << context.SS->TabletID()); - - NKikimrTxDataShard::TFlatSchemeTransaction tx; - context.SS->FillSeqNo(tx, seqNo); - auto& restore = *tx.MutableRestore(); - restore.CopyFrom(table->RestoreSettings); - restore.SetTableId(pathId.LocalPathId); - restore.SetShardNum(i); - - auto ev = MakeHolder<TEvDataShard::TEvProposeTransaction>( - NKikimrTxDataShard::TX_KIND_SCHEME, - context.SS->TabletID(), - context.Ctx.SelfID, - ui64(opId.GetTxId()), - tx.SerializeAsString(), - processingParams - ); - - context.OnComplete.BindMsgToPipe(opId, datashardId, idx, ev.Release()); - } - } - - static ui64 RequestUnits(ui64 bytes, ui64 rows) { - return TRUCalculator::BulkUpsert(bytes, rows); - } - - static void FinishStats(const TOperationId& opId, TTxState& txState, TOperationContext& context) { - if (txState.TxType != TTxState::TxRestore) { - return; - } - - Y_VERIFY(TAppData::TimeProvider.Get() != nullptr); - const ui64 ts = TAppData::TimeProvider->Now().Seconds(); - - Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); - TTableInfo::TPtr table = context.SS->Tables[txState.TargetPathId]; - - auto& restoreInfo = table->RestoreHistory[opId.GetTxId()]; - - restoreInfo.StartDateTime = txState.StartTime.Seconds(); - restoreInfo.CompletionDateTime = ts; - restoreInfo.TotalShardCount = table->GetPartitions().size(); - restoreInfo.SuccessShardCount = CountIf(txState.ShardStatuses, [](const auto& kv) { - return kv.second.Success; - }); - restoreInfo.ShardStatuses = std::move(txState.ShardStatuses); - restoreInfo.DataTotalSize = txState.DataTotalSize; - - NIceDb::TNiceDb db(context.Txc.DB); - context.SS->PersistCompletedRestore(db, opId.GetTxId(), txState, restoreInfo); - } - - static void PersistTask(const TPathId& pathId, const TTxTransaction& tx, TOperationContext& context) { - const auto& restore = tx.GetRestore(); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - TTableInfo::TPtr table = context.SS->Tables.at(pathId); - table->RestoreSettings = restore; - - NIceDb::TNiceDb db(context.Txc.DB); - db.Table<Schema::RestoreTasks>() - .Key(pathId.OwnerId, pathId.LocalPathId) - .Update(NIceDb::TUpdate<Schema::RestoreTasks::Task>(restore.SerializeAsString())); - } - - static void PersistDone(const TPathId& pathId, TOperationContext& context) { - NIceDb::TNiceDb db(context.Txc.DB); - db.Table<Schema::RestoreTasks>() - .Key(pathId.OwnerId, pathId.LocalPathId) - .Delete(); - } - - static bool NeedToBill(const TPathId&, TOperationContext&) { - return true; - } -}; - -ISubOperationBase::TPtr CreateRestore(TOperationId id, const TTxTransaction& tx) { - return new TBackupRestoreOperationBase<TRestore, TEvDataShard::TEvCancelRestore>( - TTxState::TxRestore, TPathElement::EPathState::EPathStateRestore, id, tx - ); -} - -ISubOperationBase::TPtr CreateRestore(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); - return new TBackupRestoreOperationBase<TRestore, TEvDataShard::TEvCancelRestore>( - TTxState::TxRestore, TPathElement::EPathState::EPathStateRestore, id, state - ); -} - + +struct TRestore { + static constexpr TStringBuf Name() { + return "TRestore"; + } + + static bool HasTask(const TTxTransaction& tx) { + return tx.HasRestore(); + } + + static TString GetTableName(const TTxTransaction& tx) { + return tx.GetRestore().GetTableName(); + } + + static void ProposeTx(const TOperationId& opId, TTxState& txState, TOperationContext& context) { + const auto& pathId = txState.TargetPathId; + + const auto seqNo = context.SS->StartRound(txState); + const auto& processingParams = context.SS->SelectProcessingPrarams(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + TTableInfo::TPtr table = context.SS->Tables.at(pathId); + + for (ui32 i = 0; i < txState.Shards.size(); ++i) { + const auto& idx = txState.Shards[i].Idx; + const auto& datashardId = context.SS->ShardInfos[idx].TabletID; + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Propose restore" + << ", datashard: " << datashardId + << ", opId: " << opId + << ", at schemeshard: " << context.SS->TabletID()); + + NKikimrTxDataShard::TFlatSchemeTransaction tx; + context.SS->FillSeqNo(tx, seqNo); + auto& restore = *tx.MutableRestore(); + restore.CopyFrom(table->RestoreSettings); + restore.SetTableId(pathId.LocalPathId); + restore.SetShardNum(i); + + auto ev = MakeHolder<TEvDataShard::TEvProposeTransaction>( + NKikimrTxDataShard::TX_KIND_SCHEME, + context.SS->TabletID(), + context.Ctx.SelfID, + ui64(opId.GetTxId()), + tx.SerializeAsString(), + processingParams + ); + + context.OnComplete.BindMsgToPipe(opId, datashardId, idx, ev.Release()); + } + } + + static ui64 RequestUnits(ui64 bytes, ui64 rows) { + return TRUCalculator::BulkUpsert(bytes, rows); + } + + static void FinishStats(const TOperationId& opId, TTxState& txState, TOperationContext& context) { + if (txState.TxType != TTxState::TxRestore) { + return; + } + + Y_VERIFY(TAppData::TimeProvider.Get() != nullptr); + const ui64 ts = TAppData::TimeProvider->Now().Seconds(); + + Y_VERIFY(context.SS->Tables.contains(txState.TargetPathId)); + TTableInfo::TPtr table = context.SS->Tables[txState.TargetPathId]; + + auto& restoreInfo = table->RestoreHistory[opId.GetTxId()]; + + restoreInfo.StartDateTime = txState.StartTime.Seconds(); + restoreInfo.CompletionDateTime = ts; + restoreInfo.TotalShardCount = table->GetPartitions().size(); + restoreInfo.SuccessShardCount = CountIf(txState.ShardStatuses, [](const auto& kv) { + return kv.second.Success; + }); + restoreInfo.ShardStatuses = std::move(txState.ShardStatuses); + restoreInfo.DataTotalSize = txState.DataTotalSize; + + NIceDb::TNiceDb db(context.Txc.DB); + context.SS->PersistCompletedRestore(db, opId.GetTxId(), txState, restoreInfo); + } + + static void PersistTask(const TPathId& pathId, const TTxTransaction& tx, TOperationContext& context) { + const auto& restore = tx.GetRestore(); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + TTableInfo::TPtr table = context.SS->Tables.at(pathId); + table->RestoreSettings = restore; + + NIceDb::TNiceDb db(context.Txc.DB); + db.Table<Schema::RestoreTasks>() + .Key(pathId.OwnerId, pathId.LocalPathId) + .Update(NIceDb::TUpdate<Schema::RestoreTasks::Task>(restore.SerializeAsString())); + } + + static void PersistDone(const TPathId& pathId, TOperationContext& context) { + NIceDb::TNiceDb db(context.Txc.DB); + db.Table<Schema::RestoreTasks>() + .Key(pathId.OwnerId, pathId.LocalPathId) + .Delete(); + } + + static bool NeedToBill(const TPathId&, TOperationContext&) { + return true; + } +}; + +ISubOperationBase::TPtr CreateRestore(TOperationId id, const TTxTransaction& tx) { + return new TBackupRestoreOperationBase<TRestore, TEvDataShard::TEvCancelRestore>( + TTxState::TxRestore, TPathElement::EPathState::EPathStateRestore, id, tx + ); +} + +ISubOperationBase::TPtr CreateRestore(TOperationId id, TTxState::ETxState state) { + Y_VERIFY(state != TTxState::Invalid); + return new TBackupRestoreOperationBase<TRestore, TEvDataShard::TEvCancelRestore>( + TTxState::TxRestore, TPathElement::EPathState::EPathStateRestore, id, state + ); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_rtmr.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_rtmr.cpp index 5d263f00e5e..648fcd3c4d5 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_rtmr.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_rtmr.cpp @@ -146,9 +146,9 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -169,8 +169,8 @@ public: }; class TCreateRTMR: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -184,8 +184,8 @@ class TCreateRTMR: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -201,7 +201,7 @@ class TCreateRTMR: public TSubOperation { return THolder(new TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -218,11 +218,11 @@ class TCreateRTMR: public TSubOperation { } public: - TCreateRTMR(TOperationId id, const TTxTransaction& tx) + TCreateRTMR(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateRTMR(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -235,9 +235,9 @@ public: const auto ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const auto& rtmrVolumeDescription = Transaction.GetCreateRtmrVolume(); + const auto& rtmrVolumeDescription = Transaction.GetCreateRtmrVolume(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = rtmrVolumeDescription.GetName(); const ui64 shardsToCreate = rtmrVolumeDescription.GetPartitionsCount(); @@ -272,20 +272,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeRtmrVolume, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -296,8 +296,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit(shardsToCreate) - .PathShardsLimit(shardsToCreate) - .IsValidACL(acl); + .PathShardsLimit(shardsToCreate) + .IsValidACL(acl); } if (!checks) { @@ -313,8 +313,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -408,16 +408,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateRTMR"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateRTMR"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateRTMR AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -428,12 +428,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, const TTxTransaction& tx) { - return new TCreateRTMR(id, tx); -} - +ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, const TTxTransaction& tx) { + return new TCreateRTMR(id, tx); +} + ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateRTMR(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_solomon.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_solomon.cpp index 1b288989dc0..a8211698542 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_solomon.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_solomon.cpp @@ -181,9 +181,9 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); return true; } @@ -204,8 +204,8 @@ public: }; class TCreateSolomon: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -219,8 +219,8 @@ class TCreateSolomon: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -236,7 +236,7 @@ class TCreateSolomon: public TSubOperation { return THolder(new TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -253,11 +253,11 @@ class TCreateSolomon: public TSubOperation { } public: - TCreateSolomon(TOperationId id, const TTxTransaction& tx) + TCreateSolomon(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateSolomon(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -270,9 +270,9 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const auto& solomonDescription = Transaction.GetCreateSolomonVolume(); + const auto& solomonDescription = Transaction.GetCreateSolomonVolume(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = solomonDescription.GetName(); const ui32 channelProfileId = solomonDescription.GetChannelProfileId(); @@ -313,20 +313,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeSolomonVolume, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -337,8 +337,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit(shardsToCreate) - .PathShardsLimit(shardsToCreate) - .IsValidACL(acl); + .PathShardsLimit(shardsToCreate) + .IsValidACL(acl); } if (!checks) { @@ -354,7 +354,7 @@ public: } } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -433,7 +433,7 @@ public: context.SS->PersistTxState(db, OperationId); context.SS->PersistPath(db, newSolomon->PathId); - + if (!acl.empty()) { newSolomon->ApplyACL(acl); context.SS->PersistACL(db, newSolomon); @@ -464,16 +464,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateSolomon"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateSolomon"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateSolomon AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -484,12 +484,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, const TTxTransaction& tx) { - return new TCreateSolomon(id, tx); -} - +ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, const TTxTransaction& tx) { + return new TCreateSolomon(id, tx); +} + ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateSolomon(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp index 9ad8e7d3b96..b46d7015cc8 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp @@ -35,8 +35,8 @@ void PersistShards(NIceDb::TNiceDb& db, TTxState& txState, TSchemeShard* ss) { class TCreateSubDomain: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -50,8 +50,8 @@ class TCreateSubDomain: public TSubOperation { return TTxState::ConfigureParts; case TTxState::ConfigureParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -67,7 +67,7 @@ class TCreateSubDomain: public TSubOperation { return THolder(new NSubDomainState::TConfigureParts(OperationId)); case TTxState::Propose: return THolder(new NSubDomainState::TPropose(OperationId)); - case TTxState::Done: + case TTxState::Done: return THolder(new TDone(OperationId)); default: return nullptr; @@ -84,11 +84,11 @@ class TCreateSubDomain: public TSubOperation { } public: - TCreateSubDomain(TOperationId id, const TTxTransaction& tx) + TCreateSubDomain(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TCreateSubDomain(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -101,10 +101,10 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const auto& settings = Transaction.GetSubDomain(); + const auto& settings = Transaction.GetSubDomain(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = settings.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = settings.GetName(); ui64 shardsToCreate = settings.GetCoordinators() + settings.GetMediators(); @@ -151,20 +151,20 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeSubDomain, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -175,8 +175,8 @@ public: .PathsLimit() //check capacity on root Domain .DirChildrenLimit() .PathShardsLimit(shardsToCreate) - .ShardsLimit(shardsToCreate) //check capacity on root Domain - .IsValidACL(acl); + .ShardsLimit(shardsToCreate) //check capacity on root Domain + .IsValidACL(acl); } if (!checks) { @@ -225,11 +225,11 @@ public: return result; } - if (settings.HasResourcesDomainKey()) { + if (settings.HasResourcesDomainKey()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "Resources domain key unsupported for non-external subdomains"); - return result; - } - + return result; + } + auto domainId = parentPath.DomainId(); Y_VERIFY(context.SS->PathsById.contains(domainId)); Y_VERIFY(context.SS->SubDomains.contains(domainId)); @@ -256,11 +256,11 @@ public: } } - const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); + const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); TUserAttributes::TPtr userAttrs = new TUserAttributes(1); - - TString errStr; - + + TString errStr; + if (!userAttrs->ApplyPatch(EUserAttributesOp::CreateSubDomain, userAttrsDetails, errStr) || !userAttrs->CheckLimits(errStr)) { @@ -268,7 +268,7 @@ public: return result; } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -304,8 +304,8 @@ public: TSubDomainInfo::TPtr alter = new TSubDomainInfo( 1, settings.GetPlanResolution(), - settings.GetTimeCastBucketsPerMediator(), - newNode->PathId); + settings.GetTimeCastBucketsPerMediator(), + newNode->PathId); alter->SetSchemeLimits(parentPath.DomainInfo()->GetSchemeLimits()); //inherit from root @@ -370,16 +370,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateSubDomain"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateSubDomain"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateSubDomain AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -390,12 +390,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateSubDomain(TOperationId id, const TTxTransaction& tx) { - return new TCreateSubDomain(id, tx); -} - +ISubOperationBase::TPtr CreateSubDomain(TOperationId id, const TTxTransaction& tx) { + return new TCreateSubDomain(id, tx); +} + ISubOperationBase::TPtr CreateSubDomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TCreateSubDomain(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index 2df143fd85f..50045225298 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -166,9 +166,9 @@ public: TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState->TxType == TTxState::TxCreateTable); - NKikimrTxDataShard::TFlatSchemeTransaction txTemplate; - context.SS->FillAsyncIndexInfo(txState->TargetPathId, txTemplate); - + NKikimrTxDataShard::TFlatSchemeTransaction txTemplate; + context.SS->FillAsyncIndexInfo(txState->TargetPathId, txTemplate); + txState->ClearShardsInProgress(); const ui64 subDomainPathId = context.SS->ResolveDomainId(txState->TargetPathId).LocalPathId; @@ -185,7 +185,7 @@ public: << " datashardId: " << datashardId << " seqNo: " << seqNo); - NKikimrTxDataShard::TFlatSchemeTransaction tx(txTemplate); + NKikimrTxDataShard::TFlatSchemeTransaction tx(txTemplate); auto tableDesc = tx.MutableCreateTable(); context.SS->FillSeqNo(tx, seqNo); context.SS->FillTableDescription(txState->TargetPathId, i, NEW_TABLE_ALTER_VERSION, tableDesc); @@ -272,11 +272,11 @@ public: TTableInfo::TPtr table = context.SS->Tables[pathId]; Y_VERIFY(table); table->AlterVersion = NEW_TABLE_ALTER_VERSION; - - if (table->IsTTLEnabled()) { - context.SS->TTLEnabledTables[pathId] = table; - context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); - } + + if (table->IsTTLEnabled()) { + context.SS->TTLEnabledTables[pathId] = table; + context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); + } context.SS->PersistTableCreated(db, pathId); auto parentDir = context.SS->PathsById.at(path->ParentPathId); @@ -319,7 +319,7 @@ public: class TCreateTable: public TSubOperation { const TOperationId OperationId; - const TTxTransaction Transaction; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; bool AllowShadowData = false; @@ -374,12 +374,12 @@ class TCreateTable: public TSubOperation { } public: - TCreateTable(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TCreateTable(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TCreateTable(TOperationId id, TTxState::ETxState state) : OperationId(id) , State(state) @@ -403,9 +403,9 @@ public: const TTabletId ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - auto schema = Transaction.GetCreateTable(); + auto schema = Transaction.GetCreateTable(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = schema.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -426,12 +426,12 @@ public: NSchemeShard::TPath parentPath = NSchemeShard::TPath::Resolve(parentPathStr, context.SS); { NSchemeShard::TPath::TChecker checks = parentPath.Check(); - checks + checks .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .NotUnderDeleting(); + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting(); if (checks) { if (parentPath.Base()->IsTableIndex()) { @@ -454,20 +454,20 @@ public: } ui32 shardsToCreate = TTableInfo::ShardsToCreate(schema); - const TString acl = Transaction.GetModifyACL().GetDiffACL(); + const TString acl = Transaction.GetModifyACL().GetDiffACL(); NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist(TPathElement::EPathType::EPathTypeTable, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -481,8 +481,8 @@ public: .PathsLimit() .DirChildrenLimit() .ShardsLimit(shardsToCreate) - .PathShardsLimit(shardsToCreate) - .IsValidACL(acl); + .PathShardsLimit(shardsToCreate) + .IsValidACL(acl); } if (!checks) { @@ -498,20 +498,20 @@ public: } } - if (schema.GetIsBackup()) { + if (schema.GetIsBackup()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "Cannot create table with explicit 'IsBackup' property"); - return result; - } - + return result; + } + if (parentPath.Base()->IsTableIndex()) { LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Creating private table for table index" << ", opId: " << OperationId); - - if (schema.HasTTLSettings()) { + + if (schema.HasTTLSettings()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "TTL on index table is not supported"); - return result; - } + return result; + } } auto domainInfo = parentPath.DomainInfo(); @@ -531,8 +531,8 @@ public: PrepareScheme(schema); - TString errStr; - + TString errStr; + NKikimrSchemeOp::TPartitionConfig compilationPartitionConfig; if (!TPartitionConfigMerger::ApplyChanges(compilationPartitionConfig, TPartitionConfigMerger::DefaultConfig(AppData()), schema.GetPartitionConfig(), AppData(), errStr) || !TPartitionConfigMerger::VerifyCreateParams(compilationPartitionConfig, AppData(), IsShadowDataAllowed(), errStr)) { @@ -542,8 +542,8 @@ public: schema.MutablePartitionConfig()->CopyFrom(compilationPartitionConfig); const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; - const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr, LocalSequences); + const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr, LocalSequences); if (!alterData.Get()) { result->SetError(NKikimrScheme::StatusSchemeError, errStr); return result; @@ -592,20 +592,20 @@ public: } } - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } - TUserAttributes::TPtr userAttrs = new TUserAttributes(1); - const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); + TUserAttributes::TPtr userAttrs = new TUserAttributes(1); + const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); if (!userAttrs->ApplyPatch(EUserAttributesOp::CreateTable, userAttrsDetails, errStr) || !userAttrs->CheckLimits(errStr)) { result->SetError(NKikimrScheme::StatusInvalidParameter, errStr); - return result; - } - + return result; + } + dstPath.MaterializeLeaf(owner); result->SetPathId(dstPath.Base()->PathId.LocalPathId); @@ -614,7 +614,7 @@ public: newTable->LastTxId = OperationId.GetTxId(); newTable->PathState = TPathElement::EPathState::EPathStateCreate; newTable->PathType = TPathElement::EPathType::EPathTypeTable; - newTable->UserAttrs->AlterData = userAttrs; + newTable->UserAttrs->AlterData = userAttrs; NIceDb::TNiceDb db(context.Txc.DB); @@ -638,7 +638,7 @@ public: context.SS->PersistPath(db, newTable->PathId); context.SS->ApplyAndPersistUserAttrs(db, newTable->PathId); - + if (!acl.empty()) { newTable->ApplyACL(acl); context.SS->PersistACL(db, newTable); @@ -684,16 +684,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TCreateTable"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TCreateTable"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TCreateTable AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -708,8 +708,8 @@ ISubOperationBase::TPtr CreateNewTable(TOperationId id, const TTxTransaction& tx auto obj = MakeHolder<TCreateTable>(id, tx); obj->SetLocalSequences(localSequences); return obj.Release(); -} - +} + ISubOperationBase::TPtr CreateNewTable(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state != TTxState::Invalid); return new TCreateTable(id, state); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.cpp index 7a9041ddf05..0fd238ee181 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.cpp @@ -30,14 +30,14 @@ void TStorageChanges::Apply(TSchemeShard* ss, NTabletFlatExecutor::TTransactionC ss->PersistTableIndex(db, pId); } - for (const auto& pId: AlterCdcStreams) { - ss->PersistCdcStreamAlterData(db, pId); - } - - for (const auto& pId: ApplyCdcStreams) { - ss->PersistCdcStream(db, pId); - } - + for (const auto& pId: AlterCdcStreams) { + ss->PersistCdcStreamAlterData(db, pId); + } + + for (const auto& pId: ApplyCdcStreams) { + ss->PersistCdcStream(db, pId); + } + for (const auto& pId: Tables) { ss->PersistTable(db, pId); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.h b/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.h index 77af9b51c0a..84a22a34ef0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_db_changes.h @@ -26,9 +26,9 @@ class TStorageChanges: public TSimpleRefCount<TStorageChanges> { TDeque<TPathId> AlterIndexes; TDeque<TPathId> ApplyIndexes; - TDeque<TPathId> AlterCdcStreams; - TDeque<TPathId> ApplyCdcStreams; - + TDeque<TPathId> AlterCdcStreams; + TDeque<TPathId> ApplyCdcStreams; + TDeque<TOperationId> TxStates; public: @@ -58,14 +58,14 @@ public: ApplyIndexes.push_back(pathId); } - void PersistAlterCdcStream(const TPathId& pathId) { - AlterCdcStreams.push_back(pathId); - } - - void PersistApplyCdcStream(const TPathId& pathId) { - ApplyCdcStreams.push_back(pathId); - } - + void PersistAlterCdcStream(const TPathId& pathId) { + AlterCdcStreams.push_back(pathId); + } + + void PersistApplyCdcStream(const TPathId& pathId) { + ApplyCdcStreams.push_back(pathId); + } + void PersistTxState(const TOperationId& opId) { TxStates.push_back(opId); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp index 074806fe539..e6f0322026a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp @@ -116,8 +116,8 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.OnComplete.DoneOperation(OperationId); return true; @@ -141,8 +141,8 @@ public: class TDropBlockStoreVolume: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -182,11 +182,11 @@ class TDropBlockStoreVolume: public TSubOperation { } public: - TDropBlockStoreVolume(TOperationId id, const TTxTransaction& tx) + TDropBlockStoreVolume(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TDropBlockStoreVolume(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -198,10 +198,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropBlockStoreVolume Propose" @@ -218,8 +218,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -242,8 +242,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -296,16 +296,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropBlockStoreVolume"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropBlockStoreVolume"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropBlockStorageVolume AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -330,12 +330,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropBSV(TOperationId id, const TTxTransaction& tx) { - return new TDropBlockStoreVolume(id, tx); -} - +ISubOperationBase::TPtr CreateDropBSV(TOperationId id, const TTxTransaction& tx) { + return new TDropBlockStoreVolume(id, tx); +} + ISubOperationBase::TPtr CreateDropBSV(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropBlockStoreVolume(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp index f4a449c444b..9d02b90e0dd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp @@ -1,556 +1,556 @@ -#include "schemeshard__operation_part.h" -#include "schemeshard__operation_common.h" -#include "schemeshard_impl.h" - -#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) - -namespace NKikimr { +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr { namespace NSchemeShard { - -namespace { - -class TPropose: public TSubOperationState { + +namespace { + +class TPropose: public TSubOperationState { TString DebugHint() const override { - return TStringBuilder() - << "DropCdcStream TPropose" - << " opId# " << OperationId << " "; - } - -public: - explicit TPropose(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropCdcStream); - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - const auto step = TStepId(ev->Get()->StepId); - - LOG_I(DebugHint() << "HandleReply TEvOperationPlan" - << ": step# " << step); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropCdcStream); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(!path->Dropped()); - path->SetDropped(step, OperationId.GetTxId()); - - NIceDb::TNiceDb db(context.Txc.DB); - - context.SS->PersistDropStep(db, pathId, step, OperationId); - context.SS->PersistRemoveCdcStream(db, pathId); - - Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); - auto parent = context.SS->PathsById.at(path->ParentPathId); - - context.SS->ResolveDomainInfo(pathId)->DecPathsInside(); - parent->DecAliveChildren(); - - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - return true; - } - -private: - const TOperationId OperationId; - -}; // TPropose - -class TDropCdcStream: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::Propose; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return TTxState::Done; - default: - return TTxState::Invalid; - } - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Propose: - return THolder(new TPropose(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TDropCdcStream(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TDropCdcStream(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + return TStringBuilder() + << "DropCdcStream TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropCdcStream); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropCdcStream); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(!path->Dropped()); + path->SetDropped(step, OperationId.GetTxId()); + + NIceDb::TNiceDb db(context.Txc.DB); + + context.SS->PersistDropStep(db, pathId, step, OperationId); + context.SS->PersistRemoveCdcStream(db, pathId); + + Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); + auto parent = context.SS->PathsById.at(path->ParentPathId); + + context.SS->ResolveDomainInfo(pathId)->DecPathsInside(); + parent->DecAliveChildren(); + + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TDropCdcStream: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Propose: + return THolder(new TPropose(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TDropCdcStream(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TDropCdcStream(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetDrop(); - const auto& streamName = op.GetName(); - - LOG_N("TDropCdcStream Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << streamName); - + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetDrop(); + const auto& streamName = op.GetName(); + + LOG_N("TDropCdcStream Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto streamPath = op.HasId() - ? TPath::Init(context.SS->MakeLocalId(op.GetId()), context.SS) - : TPath::Resolve(workingDir, context.SS).Dive(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - const auto tablePath = streamPath.Parent(); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .IsUnderOperation() - .IsUnderTheSameOperation(OperationId.GetTxId()); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - context.DbChanges.PersistTxState(OperationId); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropCdcStream, streamPath.Base()->PathId); - txState.State = TTxState::Propose; - txState.MinStep = TStepId(1); - - streamPath.Base()->PathState = TPathElement::EPathState::EPathStateDrop; - streamPath.Base()->DropTxId = OperationId.GetTxId(); - streamPath.Base()->LastTxId = OperationId.GetTxId(); - + + const auto streamPath = op.HasId() + ? TPath::Init(context.SS->MakeLocalId(op.GetId()), context.SS) + : TPath::Resolve(workingDir, context.SS).Dive(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + const auto tablePath = streamPath.Parent(); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .IsUnderOperation() + .IsUnderTheSameOperation(OperationId.GetTxId()); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + context.DbChanges.PersistTxState(OperationId); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropCdcStream, streamPath.Base()->PathId); + txState.State = TTxState::Propose; + txState.MinStep = TStepId(1); + + streamPath.Base()->PathState = TPathElement::EPathState::EPathStateDrop; + streamPath.Base()->DropTxId = OperationId.GetTxId(); + streamPath.Base()->LastTxId = OperationId.GetTxId(); + context.SS->TabletCounters->Simple()[COUNTER_CDC_STREAMS_COUNT].Sub(1); - context.SS->ClearDescribePathCaches(streamPath.Base()); - context.OnComplete.PublishToSchemeBoard(OperationId, streamPath.Base()->PathId); - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropCdcStream"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TDropCdcStream AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TDropCdcStream - -class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { -protected: - void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); - auto table = context.SS->Tables.at(pathId); - - auto& notice = *tx.MutableDropCdcStreamNotice(); - PathIdFromPathId(pathId, notice.MutablePathId()); - notice.SetTableSchemaVersion(table->AlterVersion + 1); - - bool found = false; - for (const auto& [_, childPathId] : path->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsCdcStream() || !childPath->PlannedToDrop()) { - continue; - } - - Y_VERIFY_S(!found, "Too many cdc streams are planned to drop" - << ": found# " << PathIdFromPathId(notice.GetStreamPathId()) - << ", another# " << childPathId); - found = true; - - PathIdFromPathId(childPathId, notice.MutableStreamPathId()); - } - } - -public: - using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; - -}; // TConfigurePartsAtTable - -class TDropCdcStreamAtTable: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::ConfigureParts; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return TTxState::Propose; - case TTxState::Propose: - return TTxState::ProposedWaitParts; - case TTxState::ProposedWaitParts: - return TTxState::Done; - default: - return TTxState::Invalid; - } - - return TTxState::Invalid; - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::Waiting: - case TTxState::ConfigureParts: - return THolder(new TConfigurePartsAtTable(OperationId)); - case TTxState::Propose: - return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); - case TTxState::ProposedWaitParts: - return THolder(new NTableState::TProposedWaitParts(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } - -public: - explicit TDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - + context.SS->ClearDescribePathCaches(streamPath.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, streamPath.Base()->PathId); + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropCdcStream"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TDropCdcStream AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TDropCdcStream + +class TConfigurePartsAtTable: public NCdcStreamState::TConfigurePartsAtTable { +protected: + void FillNotice(const TPathId& pathId, NKikimrTxDataShard::TFlatSchemeTransaction& tx, TOperationContext& context) const override { + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); + auto table = context.SS->Tables.at(pathId); + + auto& notice = *tx.MutableDropCdcStreamNotice(); + PathIdFromPathId(pathId, notice.MutablePathId()); + notice.SetTableSchemaVersion(table->AlterVersion + 1); + + bool found = false; + for (const auto& [_, childPathId] : path->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsCdcStream() || !childPath->PlannedToDrop()) { + continue; + } + + Y_VERIFY_S(!found, "Too many cdc streams are planned to drop" + << ": found# " << PathIdFromPathId(notice.GetStreamPathId()) + << ", another# " << childPathId); + found = true; + + PathIdFromPathId(childPathId, notice.MutableStreamPathId()); + } + } + +public: + using NCdcStreamState::TConfigurePartsAtTable::TConfigurePartsAtTable; + +}; // TConfigurePartsAtTable + +class TDropCdcStreamAtTable: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::ConfigureParts; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return TTxState::Propose; + case TTxState::Propose: + return TTxState::ProposedWaitParts; + case TTxState::ProposedWaitParts: + return TTxState::Done; + default: + return TTxState::Invalid; + } + + return TTxState::Invalid; + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::Waiting: + case TTxState::ConfigureParts: + return THolder(new TConfigurePartsAtTable(OperationId)); + case TTxState::Propose: + return THolder(new NCdcStreamState::TProposeAtTable(OperationId)); + case TTxState::ProposedWaitParts: + return THolder(new NTableState::TProposedWaitParts(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } + +public: + explicit TDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetDropCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamName = op.GetStreamName(); - - LOG_N("TDropCdcStreamAtTable Propose" - << ": opId# " << OperationId - << ", stream# " << workingDir << "/" << tableName << "/" << streamName); - + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetDropCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamName = op.GetStreamName(); + + LOG_N("TDropCdcStreamAtTable Propose" + << ": opId# " << OperationId + << ", stream# " << workingDir << "/" << tableName << "/" << streamName); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); - - const auto tablePath = TPath::Resolve(workingDir, context.SS).Dive(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - const auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + + const auto tablePath = TPath::Resolve(workingDir, context.SS).Dive(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + const auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); - return result; - } - - if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { + return result; + } + + if (!context.SS->CheckLocks(tablePath.Base()->PathId, Transaction, errStr)) { result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); - return result; - } - - context.DbChanges.PersistTxState(OperationId); - - Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); - auto table = context.SS->Tables.at(tablePath.Base()->PathId); - - Y_VERIFY(table->AlterVersion != 0); - Y_VERIFY(!table->AlterData); - - Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); - auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); - - Y_VERIFY(stream->AlterVersion != 0); - Y_VERIFY(!stream->AlterData); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropCdcStreamAtTable, tablePath.Base()->PathId); - txState.State = TTxState::ConfigureParts; - + return result; + } + + context.DbChanges.PersistTxState(OperationId); + + Y_VERIFY(context.SS->Tables.contains(tablePath.Base()->PathId)); + auto table = context.SS->Tables.at(tablePath.Base()->PathId); + + Y_VERIFY(table->AlterVersion != 0); + Y_VERIFY(!table->AlterData); + + Y_VERIFY(context.SS->CdcStreams.contains(streamPath.Base()->PathId)); + auto stream = context.SS->CdcStreams.at(streamPath.Base()->PathId); + + Y_VERIFY(stream->AlterVersion != 0); + Y_VERIFY(!stream->AlterData); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropCdcStreamAtTable, tablePath.Base()->PathId); + txState.State = TTxState::ConfigureParts; + tablePath.Base()->PathState = NKikimrSchemeOp::EPathStateAlter; - tablePath.Base()->LastTxId = OperationId.GetTxId(); - - for (const auto& splitOpId : table->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); - } - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropCdcStreamAtTable"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TDropCdcStreamAtTable AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TDropCdcStreamAtTable - -} // anonymous - -ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { - return new TDropCdcStream(id, tx); -} - -ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, TTxState::ETxState state) { - return new TDropCdcStream(id, state); -} - -ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { - return new TDropCdcStreamAtTable(id, tx); -} - -ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { - return new TDropCdcStreamAtTable(id, state); -} - -TVector<ISubOperationBase::TPtr> CreateDropCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { + tablePath.Base()->LastTxId = OperationId.GetTxId(); + + for (const auto& splitOpId : table->GetSplitOpsInFlight()) { + context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); + } + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropCdcStreamAtTable"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TDropCdcStreamAtTable AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TDropCdcStreamAtTable + +} // anonymous + +ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, const TTxTransaction& tx) { + return new TDropCdcStream(id, tx); +} + +ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, TTxState::ETxState state) { + return new TDropCdcStream(id, state); +} + +ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx) { + return new TDropCdcStreamAtTable(id, tx); +} + +ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state) { + return new TDropCdcStreamAtTable(id, state); +} + +TVector<ISubOperationBase::TPtr> CreateDropCdcStream(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStream); - - LOG_D("CreateDropCdcStream" - << ": opId# " << opId - << ", tx# " << tx.ShortDebugString()); - - const auto& op = tx.GetDropCdcStream(); - const auto& tableName = op.GetTableName(); - const auto& streamName = op.GetStreamName(); - - const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); - - const auto tablePath = workingDirPath.Child(tableName); - { - const auto checks = tablePath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .IsCommonSensePath() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - const auto streamPath = tablePath.Child(streamName); - { - const auto checks = streamPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsCdcStream() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); - const auto status = checks.GetStatus(&explain); - return {CreateReject(opId, status, explain)}; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(tx, errStr)) { + + LOG_D("CreateDropCdcStream" + << ": opId# " << opId + << ", tx# " << tx.ShortDebugString()); + + const auto& op = tx.GetDropCdcStream(); + const auto& tableName = op.GetTableName(); + const auto& streamName = op.GetStreamName(); + + const auto workingDirPath = TPath::Resolve(tx.GetWorkingDir(), context.SS); + + const auto tablePath = workingDirPath.Child(tableName); + { + const auto checks = tablePath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsTable() + .IsCommonSensePath() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + const auto streamPath = tablePath.Child(streamName); + { + const auto checks = streamPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .IsCdcStream() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << streamPath.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusPreconditionFailed, errStr)}; - } - - if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { + } + + if (!context.SS->CheckLocks(tablePath.Base()->PathId, tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusMultipleModifications, errStr)}; - } - - TVector<ISubOperationBase::TPtr> result; - - { + } + + TVector<ISubOperationBase::TPtr> result; + + { auto outTx = TransactionTemplate(workingDirPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamAtTable); - outTx.MutableDropCdcStream()->CopyFrom(op); - - result.push_back(CreateDropCdcStreamAtTable(NextPartId(opId, result), outTx)); - } - - { + outTx.MutableDropCdcStream()->CopyFrom(op); + + result.push_back(CreateDropCdcStreamAtTable(NextPartId(opId, result), outTx)); + } + + { auto outTx = TransactionTemplate(tablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamImpl); - outTx.MutableDrop()->SetName(streamPath.Base()->Name); - - result.push_back(CreateDropCdcStreamImpl(NextPartId(opId, result), outTx)); - } - - for (const auto& [name, pathId] : streamPath.Base()->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto implPath = context.SS->PathsById.at(pathId); - - if (implPath->Dropped()) { - continue; - } - - auto streamImpl = context.SS->PathsById.at(pathId); - Y_VERIFY(streamImpl->IsPQGroup()); - - auto outTx = TransactionTemplate(streamPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup); - outTx.MutableDrop()->SetName(name); - - result.push_back(CreateDropPQ(NextPartId(opId, result), outTx)); - } - - return result; -} - + outTx.MutableDrop()->SetName(streamPath.Base()->Name); + + result.push_back(CreateDropCdcStreamImpl(NextPartId(opId, result), outTx)); + } + + for (const auto& [name, pathId] : streamPath.Base()->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto implPath = context.SS->PathsById.at(pathId); + + if (implPath->Dropped()) { + continue; + } + + auto streamImpl = context.SS->PathsById.at(pathId); + Y_VERIFY(streamImpl->IsPQGroup()); + + auto outTx = TransactionTemplate(streamPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup); + outTx.MutableDrop()->SetName(name); + + result.push_back(CreateDropPQ(NextPartId(opId, result), outTx)); + } + + return result; +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp index 411fed4b0e5..a84868b77bc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp @@ -84,26 +84,26 @@ public: context.SS->FillSeqNo(tx, seqNo); auto notice = tx.MutableDropIndexNotice(); - PathIdFromPathId(pathId, notice->MutablePathId()); + PathIdFromPathId(pathId, notice->MutablePathId()); notice->SetTableSchemaVersion(table->AlterVersion + 1); - bool found = false; - for (const auto& [_, childPathId] : path->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsTableIndex() || !childPath->PlannedToDrop()) { - continue; - } - - Y_VERIFY_S(!found, "Too many indexes are planned to drop" - << ": found# " << PathIdFromPathId(notice->GetIndexPathId()) - << ", another# " << childPathId); - found = true; - - PathIdFromPathId(childPathId, notice->MutableIndexPathId()); - } - + bool found = false; + for (const auto& [_, childPathId] : path->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsTableIndex() || !childPath->PlannedToDrop()) { + continue; + } + + Y_VERIFY_S(!found, "Too many indexes are planned to drop" + << ": found# " << PathIdFromPathId(notice->GetIndexPathId()) + << ", another# " << childPathId); + found = true; + + PathIdFromPathId(childPathId, notice->MutableIndexPathId()); + } + Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); } @@ -507,19 +507,19 @@ TVector<ISubOperationBase::TPtr> CreateDropIndex(TOperationId nextId, const TTxT { auto mainTableIndexDropping = TransactionTemplate(workingDirPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndexAtMainTable); - auto operation = mainTableIndexDropping.MutableDropIndex(); + auto operation = mainTableIndexDropping.MutableDropIndex(); operation->SetTableName(mainTablePath.LeafName()); operation->SetIndexName(indexPath.LeafName()); - result.push_back(CreateDropTableIndexAtMainTable(NextPartId(nextId, result), mainTableIndexDropping)); + result.push_back(CreateDropTableIndexAtMainTable(NextPartId(nextId, result), mainTableIndexDropping)); } { auto indexDropping = TransactionTemplate(mainTablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); auto operation = indexDropping.MutableDrop(); operation->SetName(ToString(indexPath.Base()->Name)); - - result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping)); + + result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping)); } for (const auto& items: indexPath.Base()->GetChildren()) { @@ -535,8 +535,8 @@ TVector<ISubOperationBase::TPtr> CreateDropIndex(TOperationId nextId, const TTxT auto implTableDropping = TransactionTemplate(indexPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); auto operation = implTableDropping.MutableDrop(); operation->SetName(items.first); - - result.push_back(CreateDropTable(NextPartId(nextId, result), implTableDropping)); + + result.push_back(CreateDropTable(NextPartId(nextId, result), implTableDropping)); } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp index 0d47e68dc78..7064fabe1ea 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp @@ -250,7 +250,7 @@ public: class TDropTableIndex: public TSubOperation { const TOperationId OperationId; - const TTxTransaction Transaction; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState AfterPropose = TTxState::Invalid; @@ -298,12 +298,12 @@ class TDropTableIndex: public TSubOperation { } public: - TDropTableIndex(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TDropTableIndex(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TDropTableIndex(TOperationId id, TTxState::ETxState state) : OperationId(id) , State(state) @@ -314,31 +314,31 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetDrop().GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetDrop().GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropTableIndex Propose" << ", path: " << parentPathStr << "/" << name - << ", pathId: " << Transaction.GetDrop().GetId() + << ", pathId: " << Transaction.GetDrop().GetId() << ", operationId: " << OperationId << ", at schemeshard: " << ssId); auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); - if (!Transaction.HasDrop()) { + if (!Transaction.HasDrop()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "Drop is not present"); return result; } - TPath index = Transaction.GetDrop().HasId() + TPath index = Transaction.GetDrop().HasId() ? TPath::Init(context.SS->MakeLocalId(Transaction.GetDrop().GetId()), context.SS) : TPath::Resolve(parentPathStr, context.SS).Dive(name); { TPath::TChecker checks = index.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -359,8 +359,8 @@ public: TPath parentTable = index.Parent(); { TPath::TChecker checks = parentTable.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .IsResolved() .NotDeleted() .IsTable() @@ -404,14 +404,14 @@ public: "TDropTableIndex AbortPropose" << ", opId: " << OperationId << ", at schemeshard: " << context.SS->TabletID()); - } - + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropTableIndex AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -422,8 +422,8 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropTableIndex(TOperationId id, const TTxTransaction& tx) { - return new TDropTableIndex(id, tx); +ISubOperationBase::TPtr CreateDropTableIndex(TOperationId id, const TTxTransaction& tx) { + return new TDropTableIndex(id, tx); } ISubOperationBase::TPtr CreateDropTableIndex(TOperationId id, TTxState::ETxState state) { @@ -445,8 +445,8 @@ TVector<ISubOperationBase::TPtr> CreateDropIndexedTable(TOperationId nextId, con { TPath::TChecker checks = table.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .IsResolved() .NotDeleted() .IsTable() @@ -464,7 +464,7 @@ TVector<ISubOperationBase::TPtr> CreateDropIndexedTable(TOperationId nextId, con result->SetError(status, explain); - if (table.IsResolved() && table.Base()->IsTable() && (table.Base()->PlannedToDrop() || table.Base()->Dropped())) { + if (table.IsResolved() && table.Base()->IsTable() && (table.Base()->PlannedToDrop() || table.Base()->Dropped())) { result->SetPathDropTxId(ui64(table.Base()->DropTxId)); result->SetPathId(table.Base()->PathId.LocalPathId); } @@ -474,107 +474,107 @@ TVector<ISubOperationBase::TPtr> CreateDropIndexedTable(TOperationId nextId, con } TVector<ISubOperationBase::TPtr> result; - result.push_back(CreateDropTable(NextPartId(nextId, result), tx)); + result.push_back(CreateDropTable(NextPartId(nextId, result), tx)); - for (const auto& [childName, childPathId] : table.Base()->GetChildren()) { - TPath child = table.Child(childName); + for (const auto& [childName, childPathId] : table.Base()->GetChildren()) { + TPath child = table.Child(childName); { - TPath::TChecker checks = child.Check(); - checks - .NotEmpty() + TPath::TChecker checks = child.Check(); + checks + .NotEmpty() .IsResolved(); if (checks) { - if (child.IsDeleted()) { + if (child.IsDeleted()) { continue; } } - if (child.IsTableIndex()) { - checks.IsTableIndex(); - } else if (child.IsCdcStream()) { - checks.IsCdcStream(); + if (child.IsTableIndex()) { + checks.IsTableIndex(); + } else if (child.IsCdcStream()) { + checks.IsCdcStream(); } else if (child.IsSequence()) { checks.IsSequence(); - } - + } + checks.NotDeleted() .NotUnderDeleting() .NotUnderOperation(); if (!checks) { TString explain = TStringBuilder() << "path index fail checks" - << ", index path: " << child.PathString(); + << ", index path: " << child.PathString(); auto status = checks.GetStatus(&explain); return {CreateReject(nextId, status, explain)}; } } - Y_VERIFY(child.Base()->PathId == childPathId); + Y_VERIFY(child.Base()->PathId == childPathId); if (child.IsSequence()) { auto dropSequence = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence); - dropSequence.MutableDrop()->SetName(ToString(child->Name)); + dropSequence.MutableDrop()->SetName(ToString(child->Name)); result.push_back(CreateDropSequence(NextPartId(nextId, result), dropSequence)); continue; - } else if (child.IsTableIndex()) { - auto dropIndex = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); - dropIndex.MutableDrop()->SetName(ToString(child.Base()->Name)); - - result.push_back(CreateDropTableIndex(NextPartId(nextId, result), dropIndex)); - } else if (child.IsCdcStream()) { - auto dropStream = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamImpl); - dropStream.MutableDrop()->SetName(ToString(child.Base()->Name)); - - result.push_back(CreateDropCdcStreamImpl(NextPartId(nextId, result), dropStream)); + } else if (child.IsTableIndex()) { + auto dropIndex = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); + dropIndex.MutableDrop()->SetName(ToString(child.Base()->Name)); + + result.push_back(CreateDropTableIndex(NextPartId(nextId, result), dropIndex)); + } else if (child.IsCdcStream()) { + auto dropStream = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStreamImpl); + dropStream.MutableDrop()->SetName(ToString(child.Base()->Name)); + + result.push_back(CreateDropCdcStreamImpl(NextPartId(nextId, result), dropStream)); } - Y_VERIFY(child.Base()->GetChildren().size() == 1); - for (auto& [implName, implPathId] : child.Base()->GetChildren()) { - Y_VERIFY(implName == "indexImplTable" || implName == "streamImpl", - "unexpected name %s", implName.c_str()); + Y_VERIFY(child.Base()->GetChildren().size() == 1); + for (auto& [implName, implPathId] : child.Base()->GetChildren()) { + Y_VERIFY(implName == "indexImplTable" || implName == "streamImpl", + "unexpected name %s", implName.c_str()); - TPath implPath = child.Child(implName); + TPath implPath = child.Child(implName); { - TPath::TChecker checks = implPath.Check(); - checks - .NotEmpty() + TPath::TChecker checks = implPath.Check(); + checks + .NotEmpty() .IsResolved() .NotDeleted() .NotUnderDeleting() .NotUnderOperation(); - if (checks) { - if (implPath.Base()->IsTable()) { - checks - .IsTable() - .IsInsideTableIndexPath(); - } else if (implPath.Base()->IsPQGroup()) { - checks - .IsPQGroup() - .IsInsideCdcStreamPath(); - } - } - + if (checks) { + if (implPath.Base()->IsTable()) { + checks + .IsTable() + .IsInsideTableIndexPath(); + } else if (implPath.Base()->IsPQGroup()) { + checks + .IsPQGroup() + .IsInsideCdcStreamPath(); + } + } + if (!checks) { TString explain = TStringBuilder() << "path impl table fail checks" - << ", index path: " << implPath.PathString(); + << ", index path: " << implPath.PathString(); auto status = checks.GetStatus(&explain); return {CreateReject(nextId, status, explain)}; } } - Y_VERIFY(implPath.Base()->PathId == implPathId); - - if (implPath.Base()->IsTable()) { - auto dropIndexTable = TransactionTemplate(child.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); - dropIndexTable.MutableDrop()->SetName(ToString(implPath.Base()->Name)); - - result.push_back(CreateDropTable(NextPartId(nextId, result), dropIndexTable)); - } else if (implPath.Base()->IsPQGroup()) { - auto dropPQGroup = TransactionTemplate(child.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup); - dropPQGroup.MutableDrop()->SetName(ToString(implPath.Base()->Name)); - - result.push_back(CreateDropPQ(NextPartId(nextId, result), dropPQGroup)); + Y_VERIFY(implPath.Base()->PathId == implPathId); + + if (implPath.Base()->IsTable()) { + auto dropIndexTable = TransactionTemplate(child.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); + dropIndexTable.MutableDrop()->SetName(ToString(implPath.Base()->Name)); + + result.push_back(CreateDropTable(NextPartId(nextId, result), dropIndexTable)); + } else if (implPath.Base()->IsPQGroup()) { + auto dropPQGroup = TransactionTemplate(child.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup); + dropPQGroup.MutableDrop()->SetName(ToString(implPath.Base()->Name)); + + result.push_back(CreateDropPQ(NextPartId(nextId, result), dropPQGroup)); } } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_kesus.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_kesus.cpp index 8bdd2a51239..ad42e96de59 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_kesus.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_kesus.cpp @@ -111,8 +111,8 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.OnComplete.DoneOperation(OperationId); return true; @@ -136,8 +136,8 @@ public: class TDropKesus: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -177,11 +177,11 @@ class TDropKesus: public TSubOperation { } public: - TDropKesus(TOperationId id, const TTxTransaction& tx) + TDropKesus(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TDropKesus(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -193,10 +193,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropKesus Propose" @@ -213,8 +213,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -237,8 +237,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -284,16 +284,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropKesus"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropKesus"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropKesus AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -318,12 +318,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropKesus(TOperationId id, const TTxTransaction& tx) { - return new TDropKesus(id, tx); -} - +ISubOperationBase::TPtr CreateDropKesus(TOperationId id, const TTxTransaction& tx) { + return new TDropKesus(id, tx); +} + ISubOperationBase::TPtr CreateDropKesus(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropKesus(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp index 64f80dfcc19..c338b4c358b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp @@ -241,11 +241,11 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - context.OnComplete.ActivateTx(OperationId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + context.OnComplete.ActivateTx(OperationId); + return true; } @@ -267,8 +267,8 @@ public: class TDropPQ: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -282,8 +282,8 @@ class TDropPQ: public TSubOperation { return TTxState::DeleteParts; case TTxState::DeleteParts: return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; + case TTxState::Propose: + return TTxState::Done; default: return TTxState::Invalid; } @@ -299,8 +299,8 @@ class TDropPQ: public TSubOperation { return THolder(new TDeleteParts(OperationId)); case TTxState::Propose: return THolder(new TPropose(OperationId)); - case TTxState::Done: - return THolder(new TDone(OperationId)); + case TTxState::Done: + return THolder(new TDone(OperationId)); default: return nullptr; } @@ -316,11 +316,11 @@ class TDropPQ: public TSubOperation { } public: - TDropPQ(TOperationId id, const TTxTransaction& tx) + TDropPQ(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TDropPQ(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -366,11 +366,11 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); + + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); - NKikimrSchemeOp::EDropWaitPolicy dropPolicy = drop.HasWaitPolicy() ? drop.GetWaitPolicy() : NKikimrSchemeOp::EDropFailOnChanges; LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -388,15 +388,15 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() .NotUnderDeleting() .NotUnderOperation() - .IsPQGroup(); + .IsPQGroup(); if (checks) { if (dropPolicy == NKikimrSchemeOp::EDropFailOnChanges) { @@ -417,45 +417,45 @@ public: } } - TPath parent = path.Parent(); - { - TPath::TChecker checks = parent.Check(); - checks - .NotEmpty() - .IsResolved() - .NotDeleted(); - - if (checks) { - if (parent.Base()->IsCdcStream()) { - checks - .IsCdcStream() - .IsInsideCdcStreamPath() - .IsUnderDeleting(TEvSchemeShard::EStatus::StatusNameConflict) - .IsUnderTheSameOperation(OperationId.GetTxId()); - } else { - checks - .IsLikeDirectory() - .IsCommonSensePath() - .NotUnderDeleting(); - } - } - - if (!checks) { - TString explain = TStringBuilder() << "parent path fail checks" - << ", path: " << parent.PathString(); - auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - + TPath parent = path.Parent(); + { + TPath::TChecker checks = parent.Check(); + checks + .NotEmpty() + .IsResolved() + .NotDeleted(); + + if (checks) { + if (parent.Base()->IsCdcStream()) { + checks + .IsCdcStream() + .IsInsideCdcStreamPath() + .IsUnderDeleting(TEvSchemeShard::EStatus::StatusNameConflict) + .IsUnderTheSameOperation(OperationId.GetTxId()); + } else { + checks + .IsLikeDirectory() + .IsCommonSensePath() + .NotUnderDeleting(); + } + } + + if (!checks) { + TString explain = TStringBuilder() << "parent path fail checks" + << ", path: " << parent.PathString(); + auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + if (dropPolicy != NKikimrSchemeOp::EDropFailOnChanges) { result->SetError(NKikimrScheme::StatusInvalidParameter, "drop policy isn't supported"); return result; } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -501,16 +501,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropPQ"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropPQ"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropPQ AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -535,12 +535,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropPQ(TOperationId id, const TTxTransaction& tx) { - return new TDropPQ(id, tx); -} - +ISubOperationBase::TPtr CreateDropPQ(TOperationId id, const TTxTransaction& tx) { + return new TDropPQ(id, tx); +} + ISubOperationBase::TPtr CreateDropPQ(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropPQ(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_replication.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_replication.cpp index 466d430c14a..b45abf97c59 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_replication.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_replication.cpp @@ -1,374 +1,374 @@ -#include "schemeshard__operation_part.h" -#include "schemeshard__operation_common.h" -#include "schemeshard_impl.h" - -#include <ydb/core/tx/replication/controller/public_events.h> - -#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -#define LOG_W(stream) LOG_WARN_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) - -namespace NKikimr { -namespace NSchemeShard { - -namespace { - -class TDropParts: public TSubOperationState { - TString DebugHint() const override { - return TStringBuilder() - << "TDropReplication TDropParts" - << " opId# " << OperationId << " "; - } - -public: - explicit TDropParts(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), {}); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropReplication); - const auto& pathId = txState->TargetPathId; - - txState->ClearShardsInProgress(); - - for (const auto& shard : txState->Shards) { - Y_VERIFY(shard.TabletType == ETabletType::ReplicationController); - - Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); - const auto tabletId = context.SS->ShardInfos.at(shard.Idx).TabletID; - - auto ev = MakeHolder<NReplication::TEvController::TEvDropReplication>(); - PathIdFromPathId(pathId, ev->Record.MutablePathId()); - ev->Record.MutableOperationId()->SetTxId(ui64(OperationId.GetTxId())); - ev->Record.MutableOperationId()->SetPartId(ui32(OperationId.GetSubTxId())); - - LOG_D(DebugHint() << "Send TEvDropReplication to controller" - << ": tabletId# " << tabletId - << ", ev# " << ev->ToString()); - context.OnComplete.BindMsgToPipe(OperationId, tabletId, pathId, ev.Release()); - - txState->ShardsInProgress.insert(shard.Idx); - } - - return false; - } - - bool HandleReply(NReplication::TEvController::TEvDropReplicationResult::TPtr& ev, TOperationContext& context) override { - LOG_I(DebugHint() << "HandleReply " << ev->Get()->ToString()); - - const auto tabletId = TTabletId(ev->Get()->Record.GetOrigin()); - const auto status = ev->Get()->Record.GetStatus(); - - switch (status) { - case NKikimrReplication::TEvDropReplicationResult::SUCCESS: - case NKikimrReplication::TEvDropReplicationResult::NOT_FOUND: - break; - default: - LOG_W(DebugHint() << "Ignoring unexpected TEvDropReplicationResult" - << " tabletId# " << tabletId - << " status# " << static_cast<int>(status)); - return false; - } - - auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropReplication); - Y_VERIFY(txState->State == TTxState::DropParts); - - const auto shardIdx = context.SS->MustGetShardIdx(tabletId); - if (!txState->ShardsInProgress.erase(shardIdx)) { - LOG_W(DebugHint() << "Ignoring duplicate TEvDropReplicationResult"); - return false; - } - - context.OnComplete.UnbindMsgFromPipe(OperationId, tabletId, txState->TargetPathId); - - if (!txState->ShardsInProgress.empty()) { - return false; - } - - NIceDb::TNiceDb db(context.Txc.DB); - context.SS->ChangeTxState(db, OperationId, TTxState::Propose); - context.OnComplete.ActivateTx(OperationId); - - return true; - } - -private: - const TOperationId OperationId; - -}; // TDropParts - -class TPropose: public TSubOperationState { - TString DebugHint() const override { - return TStringBuilder() - << "TDropReplication TPropose" - << " opId# " << OperationId << " "; - } - -public: - explicit TPropose(TOperationId id) - : OperationId(id) - { - IgnoreMessages(DebugHint(), { - NReplication::TEvController::TEvDropReplicationResult::EventType, - }); - } - - bool ProgressState(TOperationContext& context) override { - LOG_I(DebugHint() << "ProgressState"); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropReplication); - - context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); - return false; - } - - bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { - const auto step = TStepId(ev->Get()->StepId); - - LOG_I(DebugHint() << "HandleReply TEvOperationPlan" - << ": step# " << step); - - const auto* txState = context.SS->FindTx(OperationId); - Y_VERIFY(txState); - Y_VERIFY(txState->TxType == TTxState::TxDropReplication); - const auto& pathId = txState->TargetPathId; - - Y_VERIFY(context.SS->PathsById.contains(pathId)); - auto path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); - auto parentPath = context.SS->PathsById.at(path->ParentPathId); - - NIceDb::TNiceDb db(context.Txc.DB); - - Y_VERIFY(!path->Dropped()); - path->SetDropped(step, OperationId.GetTxId()); - context.SS->PersistDropStep(db, pathId, step, OperationId); - context.SS->PersistReplicationRemove(db, pathId); - - context.SS->TabletCounters->Simple()[COUNTER_USER_ATTRIBUTES_COUNT].Add(-path->UserAttrs->Size()); - context.SS->PersistUserAttributes(db, path->PathId, path->UserAttrs, nullptr); - - context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(-1); - context.SS->ResolveDomainInfo(pathId)->DecPathsInside(); - parentPath->DecAliveChildren(); - - ++parentPath->DirAlterVersion; - context.SS->PersistPathDirAlterVersion(db, parentPath); - - context.SS->ClearDescribePathCaches(parentPath); - context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); - - context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - - context.SS->ChangeTxState(db, OperationId, TTxState::Done); - return true; - } - -private: - const TOperationId OperationId; - -}; // TPropose - -class TDropReplication: public TSubOperation { - static TTxState::ETxState NextState() { - return TTxState::DropParts; - } - - static TTxState::ETxState NextState(TTxState::ETxState state) { - switch (state) { - case TTxState::DropParts: - return TTxState::Propose; - case TTxState::Propose: - return TTxState::Done; - default: - return TTxState::Invalid; - } - } - - TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch (state) { - case TTxState::DropParts: - return MakeHolder<TDropParts>(OperationId); - case TTxState::Propose: - return MakeHolder<TPropose>(OperationId); - case TTxState::Done: - return MakeHolder<TDone>(OperationId); - default: - return nullptr; - } - } - - void StateDone(TOperationContext& context) override { - State = NextState(State); - - if (State != TTxState::Invalid) { - SetState(SelectStateFunc(State)); - context.OnComplete.ActivateTx(OperationId); - } - } -public: - explicit TDropReplication(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - , State(TTxState::Invalid) - { - } - - explicit TDropReplication(TOperationId id, TTxState::ETxState state) - : OperationId(id) - , State(state) - { - SetState(SelectStateFunc(state)); - } - - THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const auto& workingDir = Transaction.GetWorkingDir(); - const auto& op = Transaction.GetDrop(); - const auto& name = op.GetName(); - - LOG_N("TDropReplication Propose" - << ": opId# " << OperationId - << ", path# " << workingDir << "/" << name); - - auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(context.SS->SelfTabletId())); - - const auto path = op.HasId() - ? TPath::Init(context.SS->MakeLocalId(op.GetId()), context.SS) - : TPath::Resolve(workingDir, context.SS).Dive(name); - { - const auto checks = path.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .IsReplication() - .NotDeleted() - .NotUnderDeleting() - .NotUnderOperation(); - - if (!checks) { - TString explain = TStringBuilder() << "path checks failed, path: " << path.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - if (path.IsResolved() && path->IsReplication() && (path->PlannedToDrop() || path->Dropped())) { - result->SetPathDropTxId(ui64(path->DropTxId)); - result->SetPathId(path->PathId.LocalPathId); - } - - return result; - } - } - - const auto parentPath = path.Parent(); - { - const auto checks = parentPath.Check(); - checks - .NotEmpty() - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() - .IsLikeDirectory(); - - if (!checks) { - TString explain = TStringBuilder() << "parent path checks failed, path: " << parentPath.PathString(); - const auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - - return result; - } - } - - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { - result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); - return result; - } - - Y_VERIFY(context.SS->Replications.contains(path->PathId)); - auto replication = context.SS->Replications.at(path->PathId); - Y_VERIFY(!replication->AlterData); - - Y_VERIFY(!context.SS->FindTx(OperationId)); - auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropReplication, path->PathId); - txState.State = TTxState::DropParts; - txState.MinStep = TStepId(1); - - for (const auto& shardIdx: path.DomainInfo()->GetReplicationControllers()) { - Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx); - const auto tabletType = context.SS->ShardInfos.at(shardIdx).TabletType; - txState.Shards.emplace_back(shardIdx, tabletType, TTxState::DropParts); - } - - path->PathState = TPathElement::EPathState::EPathStateDrop; - path->DropTxId = OperationId.GetTxId(); - path->LastTxId = OperationId.GetTxId(); - - NIceDb::TNiceDb db(context.Txc.DB); - - context.SS->PersistLastTxId(db, path.Base()); - context.SS->PersistTxState(db, OperationId); - - ++parentPath->DirAlterVersion; - context.SS->PersistPathDirAlterVersion(db, parentPath.Base()); - - context.SS->ClearDescribePathCaches(parentPath.Base()); - context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); - - context.SS->ClearDescribePathCaches(path.Base()); - context.OnComplete.PublishToSchemeBoard(OperationId, path->PathId); - - context.OnComplete.ActivateTx(OperationId); - - State = NextState(); - SetState(SelectStateFunc(State)); - - return result; - } - - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropReplication"); - } - - void AbortUnsafe(TTxId txId, TOperationContext& context) override { - LOG_N("TDropReplication AbortUnsafe" - << ": opId# " << OperationId - << ", txId# " << txId); - context.OnComplete.DoneOperation(OperationId); - } - -private: - const TOperationId OperationId; - const TTxTransaction Transaction; - TTxState::ETxState State; - -}; // TDropReplication - -} // anonymous - -ISubOperationBase::TPtr CreateDropReplication(TOperationId id, const TTxTransaction& tx) { - return new TDropReplication(id, tx); -} - -ISubOperationBase::TPtr CreateDropReplication(TOperationId id, TTxState::ETxState state) { - return new TDropReplication(id, state); -} - -} // NSchemeShard -} // NKikimr +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#include <ydb/core/tx/replication/controller/public_events.h> + +#define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_W(stream) LOG_WARN_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace NKikimr { +namespace NSchemeShard { + +namespace { + +class TDropParts: public TSubOperationState { + TString DebugHint() const override { + return TStringBuilder() + << "TDropReplication TDropParts" + << " opId# " << OperationId << " "; + } + +public: + explicit TDropParts(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), {}); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropReplication); + const auto& pathId = txState->TargetPathId; + + txState->ClearShardsInProgress(); + + for (const auto& shard : txState->Shards) { + Y_VERIFY(shard.TabletType == ETabletType::ReplicationController); + + Y_VERIFY(context.SS->ShardInfos.contains(shard.Idx)); + const auto tabletId = context.SS->ShardInfos.at(shard.Idx).TabletID; + + auto ev = MakeHolder<NReplication::TEvController::TEvDropReplication>(); + PathIdFromPathId(pathId, ev->Record.MutablePathId()); + ev->Record.MutableOperationId()->SetTxId(ui64(OperationId.GetTxId())); + ev->Record.MutableOperationId()->SetPartId(ui32(OperationId.GetSubTxId())); + + LOG_D(DebugHint() << "Send TEvDropReplication to controller" + << ": tabletId# " << tabletId + << ", ev# " << ev->ToString()); + context.OnComplete.BindMsgToPipe(OperationId, tabletId, pathId, ev.Release()); + + txState->ShardsInProgress.insert(shard.Idx); + } + + return false; + } + + bool HandleReply(NReplication::TEvController::TEvDropReplicationResult::TPtr& ev, TOperationContext& context) override { + LOG_I(DebugHint() << "HandleReply " << ev->Get()->ToString()); + + const auto tabletId = TTabletId(ev->Get()->Record.GetOrigin()); + const auto status = ev->Get()->Record.GetStatus(); + + switch (status) { + case NKikimrReplication::TEvDropReplicationResult::SUCCESS: + case NKikimrReplication::TEvDropReplicationResult::NOT_FOUND: + break; + default: + LOG_W(DebugHint() << "Ignoring unexpected TEvDropReplicationResult" + << " tabletId# " << tabletId + << " status# " << static_cast<int>(status)); + return false; + } + + auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropReplication); + Y_VERIFY(txState->State == TTxState::DropParts); + + const auto shardIdx = context.SS->MustGetShardIdx(tabletId); + if (!txState->ShardsInProgress.erase(shardIdx)) { + LOG_W(DebugHint() << "Ignoring duplicate TEvDropReplicationResult"); + return false; + } + + context.OnComplete.UnbindMsgFromPipe(OperationId, tabletId, txState->TargetPathId); + + if (!txState->ShardsInProgress.empty()) { + return false; + } + + NIceDb::TNiceDb db(context.Txc.DB); + context.SS->ChangeTxState(db, OperationId, TTxState::Propose); + context.OnComplete.ActivateTx(OperationId); + + return true; + } + +private: + const TOperationId OperationId; + +}; // TDropParts + +class TPropose: public TSubOperationState { + TString DebugHint() const override { + return TStringBuilder() + << "TDropReplication TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { + IgnoreMessages(DebugHint(), { + NReplication::TEvController::TEvDropReplicationResult::EventType, + }); + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropReplication); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropReplication); + const auto& pathId = txState->TargetPathId; + + Y_VERIFY(context.SS->PathsById.contains(pathId)); + auto path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->PathsById.contains(path->ParentPathId)); + auto parentPath = context.SS->PathsById.at(path->ParentPathId); + + NIceDb::TNiceDb db(context.Txc.DB); + + Y_VERIFY(!path->Dropped()); + path->SetDropped(step, OperationId.GetTxId()); + context.SS->PersistDropStep(db, pathId, step, OperationId); + context.SS->PersistReplicationRemove(db, pathId); + + context.SS->TabletCounters->Simple()[COUNTER_USER_ATTRIBUTES_COUNT].Add(-path->UserAttrs->Size()); + context.SS->PersistUserAttributes(db, path->PathId, path->UserAttrs, nullptr); + + context.SS->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(-1); + context.SS->ResolveDomainInfo(pathId)->DecPathsInside(); + parentPath->DecAliveChildren(); + + ++parentPath->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentPath); + + context.SS->ClearDescribePathCaches(parentPath); + context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); + + context.SS->ClearDescribePathCaches(path); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TDropReplication: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::DropParts; + } + + static TTxState::ETxState NextState(TTxState::ETxState state) { + switch (state) { + case TTxState::DropParts: + return TTxState::Propose; + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { + switch (state) { + case TTxState::DropParts: + return MakeHolder<TDropParts>(OperationId); + case TTxState::Propose: + return MakeHolder<TPropose>(OperationId); + case TTxState::Done: + return MakeHolder<TDone>(OperationId); + default: + return nullptr; + } + } + + void StateDone(TOperationContext& context) override { + State = NextState(State); + + if (State != TTxState::Invalid) { + SetState(SelectStateFunc(State)); + context.OnComplete.ActivateTx(OperationId); + } + } +public: + explicit TDropReplication(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + , State(TTxState::Invalid) + { + } + + explicit TDropReplication(TOperationId id, TTxState::ETxState state) + : OperationId(id) + , State(state) + { + SetState(SelectStateFunc(state)); + } + + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetDrop(); + const auto& name = op.GetName(); + + LOG_N("TDropReplication Propose" + << ": opId# " << OperationId + << ", path# " << workingDir << "/" << name); + + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(context.SS->SelfTabletId())); + + const auto path = op.HasId() + ? TPath::Init(context.SS->MakeLocalId(op.GetId()), context.SS) + : TPath::Resolve(workingDir, context.SS).Dive(name); + { + const auto checks = path.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .IsReplication() + .NotDeleted() + .NotUnderDeleting() + .NotUnderOperation(); + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << path.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + if (path.IsResolved() && path->IsReplication() && (path->PlannedToDrop() || path->Dropped())) { + result->SetPathDropTxId(ui64(path->DropTxId)); + result->SetPathId(path->PathId.LocalPathId); + } + + return result; + } + } + + const auto parentPath = path.Parent(); + { + const auto checks = parentPath.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory(); + + if (!checks) { + TString explain = TStringBuilder() << "parent path checks failed, path: " << parentPath.PathString(); + const auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + Y_VERIFY(context.SS->Replications.contains(path->PathId)); + auto replication = context.SS->Replications.at(path->PathId); + Y_VERIFY(!replication->AlterData); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropReplication, path->PathId); + txState.State = TTxState::DropParts; + txState.MinStep = TStepId(1); + + for (const auto& shardIdx: path.DomainInfo()->GetReplicationControllers()) { + Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx); + const auto tabletType = context.SS->ShardInfos.at(shardIdx).TabletType; + txState.Shards.emplace_back(shardIdx, tabletType, TTxState::DropParts); + } + + path->PathState = TPathElement::EPathState::EPathStateDrop; + path->DropTxId = OperationId.GetTxId(); + path->LastTxId = OperationId.GetTxId(); + + NIceDb::TNiceDb db(context.Txc.DB); + + context.SS->PersistLastTxId(db, path.Base()); + context.SS->PersistTxState(db, OperationId); + + ++parentPath->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentPath.Base()); + + context.SS->ClearDescribePathCaches(parentPath.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, parentPath->PathId); + + context.SS->ClearDescribePathCaches(path.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, path->PathId); + + context.OnComplete.ActivateTx(OperationId); + + State = NextState(); + SetState(SelectStateFunc(State)); + + return result; + } + + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropReplication"); + } + + void AbortUnsafe(TTxId txId, TOperationContext& context) override { + LOG_N("TDropReplication AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << txId); + context.OnComplete.DoneOperation(OperationId); + } + +private: + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State; + +}; // TDropReplication + +} // anonymous + +ISubOperationBase::TPtr CreateDropReplication(TOperationId id, const TTxTransaction& tx) { + return new TDropReplication(id, tx); +} + +ISubOperationBase::TPtr CreateDropReplication(TOperationId id, TTxState::ETxState state) { + return new TDropReplication(id, state); +} + +} // NSchemeShard +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_solomon.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_solomon.cpp index 99ec6f0cca4..bdd9f953d43 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_solomon.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_solomon.cpp @@ -96,8 +96,8 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedDeleteParts); return true; } @@ -120,8 +120,8 @@ public: class TDropSolomon: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -161,11 +161,11 @@ class TDropSolomon: public TSubOperation { } public: - TDropSolomon(TOperationId id, const TTxTransaction& tx) + TDropSolomon(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TDropSolomon(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -177,14 +177,14 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropSolomon Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", pathId: " << drop.GetId() << ", opId: " << OperationId << ", at schemeshard: " << ssId); @@ -197,8 +197,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -221,8 +221,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -268,16 +268,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropSolomon"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropSolomon"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropSolomon AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -302,12 +302,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, const TTxTransaction& tx) { - return new TDropSolomon(id, tx); -} - +ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, const TTxTransaction& tx) { + return new TDropSolomon(id, tx); +} + ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropSolomon(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_subdomain.cpp index 58873e01759..832fd1df70a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_subdomain.cpp @@ -91,8 +91,8 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); context.SS->ClearDescribePathCaches(path); - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedDeleteParts); return true; } @@ -115,8 +115,8 @@ public: class TDropSubdomain: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -156,11 +156,11 @@ class TDropSubdomain: public TSubOperation { } public: - TDropSubdomain(TOperationId id, const TTxTransaction& tx) + TDropSubdomain(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TDropSubdomain(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -172,10 +172,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropSubdomain Propose" @@ -192,8 +192,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -222,8 +222,8 @@ public: return result; } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -266,16 +266,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropSubdomain"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropSubdomain"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropSubdomain AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -301,12 +301,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, const TTxTransaction& tx) { - return new TDropSubdomain(id, tx); -} - +ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, const TTxTransaction& tx) { + return new TDropSubdomain(id, tx); +} + ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropSubdomain(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_table.cpp index 6534af3d3f4..45730be4095 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_table.cpp @@ -120,7 +120,7 @@ public: NKikimrTxDataShard::TFlatSchemeTransaction tx; context.SS->FillSeqNo(tx, seqNo); tx.MutableDropTable()->SetId_Deprecated(pathId.LocalPathId); - PathIdFromPathId(pathId, tx.MutableDropTable()->MutablePathId()); + PathIdFromPathId(pathId, tx.MutableDropTable()->MutablePathId()); tx.MutableDropTable()->SetName(path->Name); Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); } @@ -416,7 +416,7 @@ public: THolder<TEvDataShard::TEvSchemaChangedResult> event = MakeHolder<TEvDataShard::TEvSchemaChangedResult>(); event->Record.SetTxId(ui64(OperationId.GetTxId())); - context.OnComplete.Send(ackTo, std::move(event)); + context.OnComplete.Send(ackTo, std::move(event)); return false; } @@ -505,9 +505,9 @@ private: } public: - TDropTable(TOperationId id, const TTxTransaction& tx) + TDropTable(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) + , Transaction(tx) {} TDropTable(TOperationId id, TTxState::ETxState state) @@ -520,10 +520,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropTable Propose" @@ -540,25 +540,25 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() - .IsResolved() - .NotDeleted() - .IsTable() - .NotUnderDeleting() - .NotUnderOperation(); + .IsResolved() + .NotDeleted() + .IsTable() + .NotUnderDeleting() + .NotUnderOperation(); if (!checks) { TString explain = TStringBuilder() << "path fail checks" << ", path: " << path.PathString(); auto status = checks.GetStatus(&explain); result->SetError(status, explain); - if (path.IsResolved() && path.Base()->IsTable() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) { - result->SetPathDropTxId(ui64(path.Base()->DropTxId)); - result->SetPathId(path.Base()->PathId.LocalPathId); - } + if (path.IsResolved() && path.Base()->IsTable() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) { + result->SetPathDropTxId(ui64(path.Base()->DropTxId)); + result->SetPathId(path.Base()->PathId.LocalPathId); + } return result; } } @@ -566,20 +566,20 @@ public: TPath parent = path.Parent(); { TPath::TChecker checks = parent.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .IsResolved() .NotDeleted(); if (checks) { if (parent.Base()->IsTableIndex()) { - checks - .IsTableIndex() + checks + .IsTableIndex() .IsInsideTableIndexPath() .IsUnderDeleting() .IsUnderTheSameOperation(OperationId.GetTxId()); //allow only as part of drop base table } else { - checks + checks .IsLikeDirectory() .IsCommonSensePath() .NotUnderDeleting(); @@ -595,8 +595,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -655,14 +655,14 @@ public: "TDropTable AbortPropose" << ", opId: " << OperationId << ", at schemeshard: " << context.SS->TabletID()); - } - + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropTable AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -687,10 +687,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateDropTable(TOperationId id, const TTxTransaction& tx) { - return new TDropTable(id, tx); -} - +ISubOperationBase::TPtr CreateDropTable(TOperationId id, const TTxTransaction& tx) { + return new TDropTable(id, tx); +} + ISubOperationBase::TPtr CreateDropTable(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state != TTxState::Invalid); return new TDropTable(id, state); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_unsafe.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_unsafe.cpp index ea4e73d137d..a1a9f2ccfd0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_unsafe.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_unsafe.cpp @@ -91,10 +91,10 @@ public: context.SS->ClearDescribePathCaches(parentDir); context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); - for (const TPathId pathId : pathes) { - context.OnComplete.PublishToSchemeBoard(OperationId, pathId); - } - + for (const TPathId pathId : pathes) { + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + } + context.SS->ChangeTxState(db, OperationId, TTxState::ProposedDeleteParts); return true; } @@ -122,9 +122,9 @@ public: class TDropForceUnsafe: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; - + const TOperationId OperationId; + const TTxTransaction Transaction; + TTxState::ETxState State = TTxState::Invalid; TPathElement::EPathType ExpectedType = TPathElement::EPathType::EPathTypeInvalid; @@ -164,9 +164,9 @@ class TDropForceUnsafe: public TSubOperation { } public: - TDropForceUnsafe(TOperationId id, const TTxTransaction& tx, TPathElement::EPathType expectedType) + TDropForceUnsafe(TOperationId id, const TTxTransaction& tx, TPathElement::EPathType expectedType) : OperationId(id) - , Transaction(tx) + , Transaction(tx) , ExpectedType(expectedType) {} @@ -180,10 +180,10 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = drop.GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropForceUnsafe Propose" @@ -216,8 +216,8 @@ public: { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .IsResolved() .NotRoot() .NotDeleted() @@ -232,23 +232,23 @@ public: } } - if (path.IsUnderCreating()) { - TPath parent = path.Parent(); - if (parent.IsUnderCreating()) { - TPath::TChecker checks = parent.Check(); - checks + if (path.IsUnderCreating()) { + TPath parent = path.Parent(); + if (parent.IsUnderCreating()) { + TPath::TChecker checks = parent.Check(); + checks .NotUnderTheSameOperation(path.ActiveOperation(), NKikimrScheme::StatusMultipleModifications); - - if (!checks) { - TString explain = TStringBuilder() << "parent path fail checks" - << ", path: " << parent.PathString(); - auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - return result; - } - } - } - + + if (!checks) { + TString explain = TStringBuilder() << "parent path fail checks" + << ", path: " << parent.PathString(); + auto status = checks.GetStatus(&explain); + result->SetError(status, explain); + return result; + } + } + } + if (ExpectedType != TPathElement::EPathType::EPathTypeInvalid) { if (path.Base()->PathType != ExpectedType) { result->SetError(NKikimrScheme::StatusNameConflict, "Drop on mismatch item"); @@ -256,8 +256,8 @@ public: } } - TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -314,16 +314,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TDropForceUnsafe"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TDropForceUnsafe"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TDropForceUnsafe AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -348,21 +348,21 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, const TTxTransaction& tx) { - return new TDropForceUnsafe(id, tx, TPathElement::EPathType::EPathTypeInvalid); -} - +ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, const TTxTransaction& tx) { + return new TDropForceUnsafe(id, tx, TPathElement::EPathType::EPathTypeInvalid); +} + ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropForceUnsafe(id, state); } ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, const TTxTransaction& tx) { return new TDropForceUnsafe(id, tx, TPathElement::EPathType::EPathTypeSubDomain); -} - +} + ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TDropForceUnsafe(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_finalize_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_finalize_build_index.cpp index 200338715d8..fe4d116416f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_finalize_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_finalize_build_index.cpp @@ -68,7 +68,7 @@ public: NKikimrTxDataShard::TFlatSchemeTransaction tx; auto* op = tx.MutableFinalizeBuildIndex(); - PathIdFromPathId(pathId, op->MutablePathId()); + PathIdFromPathId(pathId, op->MutablePathId()); op->SetSnapshotTxId(ui64(snapshotTxId)); op->SetSnapshotStep(ui64(snapshotStepid)); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_initiate_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_initiate_build_index.cpp index e66db80d2ba..f8d5ca96c69 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_initiate_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_initiate_build_index.cpp @@ -52,45 +52,45 @@ public: Y_VERIFY(txState->TxType == TTxState::TxInitializeBuildIndex); TPathId pathId = txState->TargetPathId; - Y_VERIFY(context.SS->PathsById.contains(pathId)); - TPathElement::TPtr path = context.SS->PathsById.at(pathId); - - Y_VERIFY(context.SS->Tables.contains(pathId)); + Y_VERIFY(context.SS->PathsById.contains(pathId)); + TPathElement::TPtr path = context.SS->PathsById.at(pathId); + + Y_VERIFY(context.SS->Tables.contains(pathId)); const TTableInfo::TPtr tableInfo = context.SS->Tables.at(pathId); - NKikimrTxDataShard::TFlatSchemeTransaction txTemplate; - auto initiate = txTemplate.MutableInitiateBuildIndex(); - PathIdFromPathId(pathId, initiate->MutablePathId()); - initiate->SetSnapshotName("Snapshot0"); - initiate->SetTableSchemaVersion(tableInfo->AlterVersion + 1); - - bool found = false; - for (const auto& [childName, childPathId] : path->GetChildren()) { - Y_VERIFY(context.SS->PathsById.contains(childPathId)); - auto childPath = context.SS->PathsById.at(childPathId); - - if (!childPath->IsTableIndex() || childPath->Dropped() || childPath->PlannedToDrop()) { - continue; - } - - Y_VERIFY(context.SS->Indexes.contains(childPathId)); - auto index = context.SS->Indexes.at(childPathId); - - if (index->State != TTableIndexInfo::EState::EIndexStateInvalid) { - // doesn't exist yet so its state is invalid - continue; - } - - Y_VERIFY_S(!found, "Too many indexes are planned to create" - << ": found# " << TPathId(initiate->GetIndexDescription().GetPathOwnerId(), - initiate->GetIndexDescription().GetLocalPathId()) - << ", another# " << childPathId); - found = true; - - Y_VERIFY(index->AlterData); - context.SS->DescribeTableIndex(childPathId, childName, index->AlterData, *initiate->MutableIndexDescription()); - } - + NKikimrTxDataShard::TFlatSchemeTransaction txTemplate; + auto initiate = txTemplate.MutableInitiateBuildIndex(); + PathIdFromPathId(pathId, initiate->MutablePathId()); + initiate->SetSnapshotName("Snapshot0"); + initiate->SetTableSchemaVersion(tableInfo->AlterVersion + 1); + + bool found = false; + for (const auto& [childName, childPathId] : path->GetChildren()) { + Y_VERIFY(context.SS->PathsById.contains(childPathId)); + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsTableIndex() || childPath->Dropped() || childPath->PlannedToDrop()) { + continue; + } + + Y_VERIFY(context.SS->Indexes.contains(childPathId)); + auto index = context.SS->Indexes.at(childPathId); + + if (index->State != TTableIndexInfo::EState::EIndexStateInvalid) { + // doesn't exist yet so its state is invalid + continue; + } + + Y_VERIFY_S(!found, "Too many indexes are planned to create" + << ": found# " << TPathId(initiate->GetIndexDescription().GetPathOwnerId(), + initiate->GetIndexDescription().GetLocalPathId()) + << ", another# " << childPathId); + found = true; + + Y_VERIFY(index->AlterData); + context.SS->DescribeTableIndex(childPathId, childName, index->AlterData, *initiate->MutableIndexDescription()); + } + txState->ClearShardsInProgress(); for (ui32 i = 0; i < txState->Shards.size(); ++i) { @@ -99,7 +99,7 @@ public: auto seqNo = context.SS->StartRound(*txState); - NKikimrTxDataShard::TFlatSchemeTransaction tx(txTemplate); + NKikimrTxDataShard::TFlatSchemeTransaction tx(txTemplate); context.SS->FillSeqNo(tx, seqNo); TString txBody; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_just_reject.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_just_reject.cpp index 0e9443d7a9f..43bec868a2c 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_just_reject.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_just_reject.cpp @@ -27,10 +27,10 @@ public: Response->SetError(status, explain); } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TReject"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TReject"); + } + void ProgressState(TOperationContext&) override { Y_FAIL("no ProgressState for TReject"); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp index 114c22e7bab..e4151fd8dfb 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp @@ -64,11 +64,11 @@ void TMemoryChanges::GrabNewIndex(TSchemeShard* ss, const TPathId& pathId) { } void TMemoryChanges::GrabNewCdcStream(TSchemeShard* ss, const TPathId& pathId) { - Y_VERIFY(!ss->CdcStreams.contains(pathId)); - + Y_VERIFY(!ss->CdcStreams.contains(pathId)); + CdcStreams.emplace(pathId, nullptr); -} - +} + void TMemoryChanges::UnDo(TSchemeShard* ss) { // be aware of the order of grab & undo ops // stack is the best way to manage it right @@ -95,14 +95,14 @@ void TMemoryChanges::UnDo(TSchemeShard* ss) { while (CdcStreams) { const auto& [id, elem] = CdcStreams.top(); - if (elem) { - ss->CdcStreams[id] = elem; - } else { - ss->CdcStreams.erase(id); - } + if (elem) { + ss->CdcStreams[id] = elem; + } else { + ss->CdcStreams.erase(id); + } CdcStreams.pop(); - } - + } + while (Tables) { const auto& [id, elem] = Tables.top(); if (elem) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h index 32d1b939da4..c6ec264ffe7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h @@ -21,9 +21,9 @@ class TMemoryChanges: public TSimpleRefCount<TMemoryChanges> { using TIndexState = std::pair<TPathId, TTableIndexInfo::TPtr>; TStack<TIndexState> Indexes; - using TCdcStreamState = std::pair<TPathId, TCdcStreamInfo::TPtr>; + using TCdcStreamState = std::pair<TPathId, TCdcStreamInfo::TPtr>; TStack<TCdcStreamState> CdcStreams; - + using TTableState = std::pair<TPathId, TTableInfo::TPtr>; TStack<TTableState> Tables; @@ -55,7 +55,7 @@ public: void GrabNewIndex(TSchemeShard* ss, const TPathId& pathId); void GrabNewCdcStream(TSchemeShard* ss, const TPathId& pathId); - + void UnDo(TSchemeShard* ss); }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_mkdir.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_mkdir.cpp index 821efbdd4cd..a2afe4741af 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_mkdir.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_mkdir.cpp @@ -71,8 +71,8 @@ public: }; class TMkDir: public TSubOperation { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -112,29 +112,29 @@ class TMkDir: public TSubOperation { } public: - TMkDir(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TMkDir(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TMkDir(TOperationId id, TTxState::ETxState state) : OperationId(id) , State(state) - { + { SetState(SelectStateFunc(state)); - } + } THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { const auto ssId = context.SS->SelfTabletId(); const auto acceptExisted = !Transaction.GetFailOnExist(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetMkDir().GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetMkDir().GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TMkDir Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", operationId: " << OperationId << ", at schemeshard: " << ssId); @@ -162,15 +162,15 @@ public: } } - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + NSchemeShard::TPath dstPath = parentPath.Child(name); { NSchemeShard::TPath::TChecker checks = dstPath.Check(); checks.IsAtLocalSchemeShard(); if (dstPath.IsResolved()) { - checks - .IsResolved() + checks + .IsResolved() .NotUnderDeleting() .FailOnExist({ TPathElement::EPathType::EPathTypeDir, @@ -179,8 +179,8 @@ public: TPathElement::EPathType::EPathTypeColumnStore }, acceptExisted); } else { - checks - .NotEmpty() + checks + .NotEmpty() .NotResolved(); } @@ -189,8 +189,8 @@ public: .IsValidLeafName() .DepthLimit() .PathsLimit() - .DirChildrenLimit() - .IsValidACL(acl); + .DirChildrenLimit() + .IsValidACL(acl); } if (!checks) { @@ -206,15 +206,15 @@ public: } } - TString errStr; - - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } TUserAttributes::TPtr userAttrs = new TUserAttributes(1); - const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); + const auto& userAttrsDetails = Transaction.GetAlterUserAttributes(); if (!userAttrs->ApplyPatch(EUserAttributesOp::MkDir, userAttrsDetails, errStr) || !userAttrs->CheckLimits(errStr)) { @@ -267,19 +267,19 @@ public: return result; } - void AbortPropose(TOperationContext& context) override { - LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "MkDir AbortPropose" - << ", opId: " << OperationId - << ", at schemeshard: " << context.SS->TabletID()); - } - + void AbortPropose(TOperationContext& context) override { + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "MkDir AbortPropose" + << ", opId: " << OperationId + << ", at schemeshard: " << context.SS->TabletID()); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TMkDir AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -290,10 +290,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateMkDir(TOperationId id, const TTxTransaction& tx) { - return new TMkDir(id, tx); -} - +ISubOperationBase::TPtr CreateMkDir(TOperationId id, const TTxTransaction& tx) { + return new TMkDir(id, tx); +} + ISubOperationBase::TPtr CreateMkDir(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state != TTxState::Invalid); return new TMkDir(id, state); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_modify_acl.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_modify_acl.cpp index 38dab5aaf33..fac70a77047 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_modify_acl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_modify_acl.cpp @@ -8,49 +8,49 @@ using namespace NSchemeShard; class TModifyACL: public ISubOperationBase { private: - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; public: - TModifyACL(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TModifyACL(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TModifyACL(TOperationId id) : OperationId(id) - { - } + { + } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = Transaction.GetModifyACL().GetName(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& name = Transaction.GetModifyACL().GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TModifyACL Propose" - << ", path: " << parentPathStr << "/" << name + << ", path: " << parentPathStr << "/" << name << ", operationId: " << OperationId << ", at schemeshard: " << ssId); auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusSuccess, ui64(OperationId.GetTxId()), ui64(ssId)); - const TString acl = Transaction.GetModifyACL().GetDiffACL(); - + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + TPath path = TPath::Resolve(parentPathStr, context.SS).Dive(name); { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() - .IsValidACL(acl); + .IsCommonSensePath() + .IsValidACL(acl); if (!checks) { TString explain = TStringBuilder() << "path table fail checks" @@ -61,9 +61,9 @@ public: } } - TString errStr; - - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + TString errStr; + + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -107,7 +107,7 @@ public: context.SS->PersistPathDirAlterVersion(db, path.Base()); context.OnComplete.PublishToSchemeBoard(OperationId, path.Base()->PathId); - + TPath parent = path.Parent(); // we show owner in children listing, so we have to update it ++parent.Base()->DirAlterVersion; context.SS->PersistPathDirAlterVersion(db, parent.Base()); @@ -115,16 +115,16 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, parent.Base()->PathId); context.OnComplete.UpdateTenants({path.Base()->PathId}); - } + } context.OnComplete.DoneOperation(OperationId); return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TModifyACL"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TModifyACL"); + } + void ProgressState(TOperationContext&) override { Y_FAIL("no progress state for modify acl"); } @@ -139,10 +139,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateModifyACL(TOperationId id, const TTxTransaction& tx) { - return new TModifyACL(id, tx); -} - +ISubOperationBase::TPtr CreateModifyACL(TOperationId id, const TTxTransaction& tx) { + return new TModifyACL(id, tx); +} + ISubOperationBase::TPtr CreateModifyACL(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state == TTxState::Invalid); return new TModifyACL(id); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp index c0551fd31aa..fc39ce04ba3 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp @@ -89,10 +89,10 @@ public: NKikimrTxDataShard::TFlatSchemeTransaction tx; context.SS->FillSeqNo(tx, seqNo); auto move = tx.MutableMoveTable(); - PathIdFromPathId(srcPath->PathId, move->MutablePathId()); + PathIdFromPathId(srcPath->PathId, move->MutablePathId()); move->SetTableSchemaVersion(srcTable->AlterVersion+1); - PathIdFromPathId(dstPath->PathId, move->MutableDstPathId()); + PathIdFromPathId(dstPath->PathId, move->MutableDstPathId()); move->SetDstPath(TPath::Init(dstPath->PathId, context.SS).PathString()); for (const auto& child: srcPath->GetChildren()) { @@ -109,8 +109,8 @@ public: Y_VERIFY(dstIndexPath.IsResolved()); auto remap = move->AddReMapIndexes(); - PathIdFromPathId(srcIndexPath->PathId, remap->MutablePathId()); - PathIdFromPathId(dstIndexPath->PathId, remap->MutableDstPathId()); + PathIdFromPathId(srcIndexPath->PathId, remap->MutablePathId()); + PathIdFromPathId(dstIndexPath->PathId, remap->MutableDstPathId()); } Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp index 8902a01c486..1e769353310 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp @@ -1,6 +1,6 @@ #include "schemeshard__operation_part.h" #include "schemeshard_impl.h" -#include "schemeshard_path_describer.h" +#include "schemeshard_path_describer.h" namespace NKikimr { namespace NSchemeShard { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index 5d4593e071c..4a639e63429 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -1,7 +1,7 @@ #pragma once #include "schemeshard.h" -#include "schemeshard_private.h" +#include "schemeshard_private.h" #include "schemeshard_tx_infly.h" #include "schemeshard_types.h" #include "schemeshard__operation_side_effects.h" @@ -13,7 +13,7 @@ #include <ydb/core/persqueue/events/global.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/columnshard/columnshard.h> -#include <ydb/core/tx/replication/controller/public_events.h> +#include <ydb/core/tx/replication/controller/public_events.h> #include <ydb/core/tx/sequenceshard/public/events.h> #include <ydb/core/tx/tx_processing.h> @@ -46,9 +46,9 @@ action(NSequenceShard::TEvSequenceShard::TEvRestoreSequenceResult, NSchemeShard::TXTYPE_SEQUENCESHARD_RESTORE_SEQUENCE_RESULT) \ action(NSequenceShard::TEvSequenceShard::TEvRedirectSequenceResult, NSchemeShard::TXTYPE_SEQUENCESHARD_REDIRECT_SEQUENCE_RESULT) \ \ - action(NReplication::TEvController::TEvCreateReplicationResult, NSchemeShard::TXTYPE_CREATE_REPLICATION_RESULT) \ - action(NReplication::TEvController::TEvDropReplicationResult, NSchemeShard::TXTYPE_DROP_REPLICATION_RESULT) \ -\ + action(NReplication::TEvController::TEvCreateReplicationResult, NSchemeShard::TXTYPE_CREATE_REPLICATION_RESULT) \ + action(NReplication::TEvController::TEvDropReplicationResult, NSchemeShard::TXTYPE_DROP_REPLICATION_RESULT) \ +\ action(TEvSubDomain::TEvConfigureStatus, NSchemeShard::TXTYPE_SUBDOMAIN_CONFIGURE_RESULT) \ \ action(TEvSchemeShard::TEvInitTenantSchemeShardResult, NSchemeShard::TXTYPE_SUBDOMAIN_CONFIGURE_RESULT) \ @@ -116,9 +116,9 @@ public: virtual THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) = 0; - // call it inside multipart operations after failed propose - virtual void AbortPropose(TOperationContext& context) = 0; - + // call it inside multipart operations after failed propose + virtual void AbortPropose(TOperationContext& context) = 0; + //call it only before execute ForceDrop operaion for path virtual void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) = 0; }; @@ -187,35 +187,35 @@ public: ISubOperationBase::TPtr CreateReject(TOperationId id, THolder<TProposeResponse> response); ISubOperationBase::TPtr CreateReject(TOperationId id, NKikimrScheme::EStatus status, const TString& message); -ISubOperationBase::TPtr CreateMkDir(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateMkDir(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateRmDir(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateRmDir(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateModifyACL(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateModifyACL(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, TTxState::ETxState state); - +ISubOperationBase::TPtr CreateMkDir(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateMkDir(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateRmDir(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateRmDir(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateModifyACL(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateModifyACL(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterUserAttrs(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateFroceDropUnsafe(TOperationId id, TTxState::ETxState state); + ISubOperationBase::TPtr CreateNewTable(TOperationId id, const TTxTransaction& tx, const THashSet<TString>& localSequences = { }); ISubOperationBase::TPtr CreateNewTable(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateCopyTable(TOperationId id, const TTxTransaction& tx); + +ISubOperationBase::TPtr CreateCopyTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateCopyTable(TOperationId id, TTxState::ETxState state); -TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context); - -ISubOperationBase::TPtr CreateAlterTable(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterTable(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateDropTable(TOperationId id, const TTxTransaction& tx); +TVector<ISubOperationBase::TPtr> CreateCopyTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context); + +ISubOperationBase::TPtr CreateAlterTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterTable(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateDropTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateDropTable(TOperationId id, TTxState::ETxState state); TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId id, const TTxTransaction& tx, TOperationContext& context); @@ -226,38 +226,38 @@ TVector<ISubOperationBase::TPtr> CreateDropIndex(TOperationId id, const TTxTrans ISubOperationBase::TPtr CreateDropTableIndexAtMainTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateDropTableIndexAtMainTable(TOperationId id, TTxState::ETxState state); -/// CDC -// Create -TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); -ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state); -// Alter -TVector<ISubOperationBase::TPtr> CreateAlterCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); -ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state); -// Drop -TVector<ISubOperationBase::TPtr> CreateDropCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); -ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateBackup(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateBackup(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateRestore(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateRestore(TOperationId id, TTxState::ETxState state); - +/// CDC +// Create +TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); +ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewCdcStreamImpl(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewCdcStreamAtTable(TOperationId id, TTxState::ETxState state); +// Alter +TVector<ISubOperationBase::TPtr> CreateAlterCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); +ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterCdcStreamImpl(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterCdcStreamAtTable(TOperationId id, TTxState::ETxState state); +// Drop +TVector<ISubOperationBase::TPtr> CreateDropCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); +ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropCdcStreamImpl(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropCdcStreamAtTable(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateBackup(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateBackup(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateRestore(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateRestore(TOperationId id, TTxState::ETxState state); + ISubOperationBase::TPtr CreateTxCancelTx(TEvSchemeShard::TEvCancelTx::TPtr ev); - + TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context); TVector<ISubOperationBase::TPtr> CreateDropIndexedTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context); -ISubOperationBase::TPtr CreateNewTableIndex(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewTableIndex(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateNewTableIndex(TOperationId id, TTxState::ETxState state); ISubOperationBase::TPtr CreateDropTableIndex(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateDropTableIndex(TOperationId id, TTxState::ETxState state); @@ -280,33 +280,33 @@ ISubOperationBase::TPtr CreateAlterOlapTable(TOperationId id, TTxState::ETxState ISubOperationBase::TPtr CreateDropOlapTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateDropOlapTable(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateNewBSV(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewBSV(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateNewBSV(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewBSV(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterBSV(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAssignBSV(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropBSV(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropBSV(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateDropBSV(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropBSV(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateNewPQ(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewPQ(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateNewPQ(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewPQ(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterPQ(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropPQ(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropPQ(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateDropPQ(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropPQ(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateSubDomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateSubDomain(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateSubDomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateSubDomain(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterSubDomain(TOperationId id, TTxState::ETxState state); - ISubOperationBase::TPtr CreateCompatibleSubdomainDrop(TSchemeShard* ss, TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateCompatibleSubdomainAlter(TSchemeShard* ss, TOperationId id, const TTxTransaction& tx); @@ -315,42 +315,42 @@ ISubOperationBase::TPtr CreateUpgradeSubDomain(TOperationId id, TTxState::ETxSta ISubOperationBase::TPtr CreateUpgradeSubDomainDecision(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateUpgradeSubDomainDecision(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, TTxState::ETxState state); - +ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropSubdomain(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateFroceDropSubDomain(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateExtSubDomain(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterExtSubDomain(TOperationId id, TTxState::ETxState state); + ISubOperationBase::TPtr CreateFroceDropExtSubDomain(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateFroceDropExtSubDomain(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateNewKesus(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewKesus(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateDropKesus(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropKesus(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, TTxState::ETxState state); - -ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, TTxState::ETxState state); - +ISubOperationBase::TPtr CreateNewKesus(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewKesus(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateAlterKesus(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateDropKesus(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropKesus(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewRTMR(TOperationId id, TTxState::ETxState state); + +ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewSolomon(TOperationId id, TTxState::ETxState state); + ISubOperationBase::TPtr CreateAlterSolomon(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateAlterSolomon(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state); - +ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state); + ISubOperationBase::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, TTxState::ETxState state); @@ -394,10 +394,10 @@ ISubOperationBase::TPtr CreateNewSequence(TOperationId id, TTxState::ETxState st ISubOperationBase::TPtr CreateDropSequence(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateDropSequence(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateNewReplication(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateNewReplication(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateDropReplication(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateDropReplication(TOperationId id, TTxState::ETxState state); - +ISubOperationBase::TPtr CreateNewReplication(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewReplication(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateDropReplication(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropReplication(TOperationId id, TTxState::ETxState state); + } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_rmdir.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_rmdir.cpp index 67312a12f91..f2dea8f26df 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_rmdir.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_rmdir.cpp @@ -7,27 +7,27 @@ using namespace NKikimr; using namespace NSchemeShard; class TRmDir: public ISubOperationBase { - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; public: - TRmDir(TOperationId id, const TTxTransaction& tx) - : OperationId(id) - , Transaction(tx) - { - } - + TRmDir(TOperationId id, const TTxTransaction& tx) + : OperationId(id) + , Transaction(tx) + { + } + TRmDir(TOperationId id) : OperationId(id) - { - } + { + } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& drop = Transaction.GetDrop(); + const auto& drop = Transaction.GetDrop(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = drop.GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -44,8 +44,8 @@ public: : TPath::Resolve(parentPathStr, context.SS).Dive(name); { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -62,17 +62,17 @@ public: << ", path: " << path.PathString(); auto status = checks.GetStatus(&explain); result->SetError(status, explain); - if (path.IsResolved() && path.Base()->IsDirectory() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) { - result->SetPathDropTxId(ui64(path.Base()->DropTxId)); - result->SetPathId(path.Base()->PathId.LocalPathId); - } + if (path.IsResolved() && path.Base()->IsDirectory() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) { + result->SetPathDropTxId(ui64(path.Base()->DropTxId)); + result->SetPathId(path.Base()->PathId.LocalPathId); + } return result; } } - TString errStr; + TString errStr; - if (!context.SS->CheckApplyIf(Transaction, errStr)) { + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; } @@ -104,10 +104,10 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TRmDir"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TRmDir"); + } + void ProgressState(TOperationContext& context) override { LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TRmDir ProgressState" @@ -186,7 +186,7 @@ public: "RmDir AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); context.OnComplete.DoneOperation(OperationId); } @@ -197,10 +197,10 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateRmDir(TOperationId id, const TTxTransaction& tx) { - return new TRmDir(id, tx); -} - +ISubOperationBase::TPtr CreateRmDir(TOperationId id, const TTxTransaction& tx) { + return new TRmDir(id, tx); +} + ISubOperationBase::TPtr CreateRmDir(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state == TTxState::Invalid || state == TTxState::Propose); return new TRmDir(id); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp index 08292514eae..645d7f37e19 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp @@ -268,34 +268,34 @@ public: } } - auto oldAggrStats = tableInfo->GetStats().Aggregated; - + auto oldAggrStats = tableInfo->GetStats().Aggregated; + // Delete the whole old partitioning and persist the whole new partitionig as the indexes have changed context.SS->DeleteTablePartitioning(db, tableId, tableInfo); context.SS->SetPartitioning(tableId, tableInfo, std::move(newPartitioning)); context.SS->PersistTablePartitioning(db, tableId, tableInfo); - context.SS->PersistTablePartitionStats(db, tableId, tableInfo); + context.SS->PersistTablePartitionStats(db, tableId, tableInfo); context.SS->TabletCounters->Simple()[COUNTER_TABLE_SHARD_ACTIVE_COUNT].Sub(allSrcShardIdxs.size()); context.SS->TabletCounters->Simple()[COUNTER_TABLE_SHARD_INACTIVE_COUNT].Add(allSrcShardIdxs.size()); if (!tableInfo->IsBackup && !tableInfo->IsShardsStatsDetached()) { - auto newAggrStats = tableInfo->GetStats().Aggregated; - auto subDomainId = context.SS->ResolveDomainId(tableId); - auto subDomainInfo = context.SS->ResolveDomainInfo(tableId); + auto newAggrStats = tableInfo->GetStats().Aggregated; + auto subDomainId = context.SS->ResolveDomainId(tableId); + auto subDomainInfo = context.SS->ResolveDomainInfo(tableId); subDomainInfo->AggrDiskSpaceUsage(context.SS, newAggrStats, oldAggrStats); if (subDomainInfo->CheckDiskSpaceQuotas(context.SS)) { - context.SS->PersistSubDomainState(db, subDomainId, *subDomainInfo); - context.OnComplete.PublishToSchemeBoard(OperationId, subDomainId); - } + context.SS->PersistSubDomainState(db, subDomainId, *subDomainInfo); + context.OnComplete.PublishToSchemeBoard(OperationId, subDomainId); + } } - + Y_VERIFY(txState->ShardsInProgress.empty(), "All shards should have already completed their steps"); context.SS->ChangeTxState(db, OperationId, TTxState::NotifyPartitioningChanged); context.OnComplete.ActivateTx(OperationId); - context.OnComplete.PublishToSchemeBoard(OperationId, tableId); - + context.OnComplete.PublishToSchemeBoard(OperationId, tableId); + return true; } @@ -463,8 +463,8 @@ public: class TSplitMerge: public TSubOperation { private: - const TOperationId OperationId; - const TTxTransaction Transaction; + const TOperationId OperationId; + const TTxTransaction Transaction; TTxState::ETxState State = TTxState::Invalid; TTxState::ETxState NextState() { @@ -511,11 +511,11 @@ private: } public: - TSplitMerge(TOperationId id, const TTxTransaction& tx) + TSplitMerge(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) - { - } + , Transaction(tx) + { + } TSplitMerge(TOperationId id, TTxState::ETxState state) : OperationId(id) @@ -709,7 +709,7 @@ public: THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { const TTabletId ssId = context.SS->SelfTabletId(); - const auto& info = Transaction.GetSplitMergeTablePartitions(); + const auto& info = Transaction.GetSplitMergeTablePartitions(); const ui64 dstCount = info.SplitBoundarySize() + 1; const ui64 srcCount = info.SourceTabletIdSize(); @@ -742,8 +742,8 @@ public: : TPath::Resolve(info.GetTablePath(), context.SS); { TPath::TChecker checks = path.Check(); - checks - .NotEmpty() + checks + .NotEmpty() .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() @@ -978,16 +978,16 @@ public: return result; } - void AbortPropose(TOperationContext&) override { - Y_FAIL("no AbortPropose for TSplitMerge"); - } - + void AbortPropose(TOperationContext&) override { + Y_FAIL("no AbortPropose for TSplitMerge"); + } + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TSplitMerge AbortUnsafe" << ", opId: " << OperationId << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); + << ", at schemeshard: " << context.SS->TabletID()); TTxState* txState = context.SS->FindTx(OperationId); Y_VERIFY(txState); @@ -1011,12 +1011,12 @@ public: namespace NKikimr { namespace NSchemeShard { -ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, const TTxTransaction& tx) { - return new TSplitMerge(id, tx); -} - +ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, const TTxTransaction& tx) { + return new TSplitMerge(id, tx); +} + ISubOperationBase::TPtr CreateSplitMerge(TOperationId id, TTxState::ETxState state) { - Y_VERIFY(state != TTxState::Invalid); + Y_VERIFY(state != TTxState::Invalid); return new TSplitMerge(id, state); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index ae19c4d2671..ad9c6808fca 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -350,7 +350,7 @@ public: case NKikimrSchemeOp::EPathType::EPathTypeColumnTable: case NKikimrSchemeOp::EPathType::EPathTypeCdcStream: case NKikimrSchemeOp::EPathType::EPathTypeSequence: - case NKikimrSchemeOp::EPathType::EPathTypeReplication: + case NKikimrSchemeOp::EPathType::EPathTypeReplication: Y_FAIL("UNIMPLEMENTED"); case NKikimrSchemeOp::EPathType::EPathTypeInvalid: Y_UNREACHABLE(); @@ -415,7 +415,7 @@ public: path.Base()->Owner, path.GetEffectiveACL(), path.GetEffectiveACLVersion(), processing, storagePools, path.Base()->UserAttrs->Attrs, path.Base()->UserAttrs->AlterVersion, - schemeLimits, ui64(InvalidTabletId)); + schemeLimits, ui64(InvalidTabletId)); event->Record.SetInitiateMigration(true); Y_VERIFY(1 == txState->Shards.size()); diff --git a/ydb/core/tx/schemeshard/schemeshard__publish_to_scheme_board.cpp b/ydb/core/tx/schemeshard/schemeshard__publish_to_scheme_board.cpp index 7db60e175db..cb9a270d256 100644 --- a/ydb/core/tx/schemeshard/schemeshard__publish_to_scheme_board.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__publish_to_scheme_board.cpp @@ -1,121 +1,121 @@ -#include "schemeshard_impl.h" -#include "schemeshard_path_describer.h" - -namespace NKikimr { +#include "schemeshard_impl.h" +#include "schemeshard_path_describer.h" + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TTxPublishToSchemeBoard: public TSchemeShard::TRwTxBase { - THashMap<TTxId, TDeque<TPathId>> Paths; + THashMap<TTxId, TDeque<TPathId>> Paths; THashMap<TTxId, TVector<THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder>>> Descriptions; - - TTxPublishToSchemeBoard(TSelf *self, THashMap<TTxId, TDeque<TPathId>>&& paths) - : TRwTxBase(self) - , Paths(std::move(paths)) - { - } - - TTxType GetTxType() const override { - return TXTYPE_DEFERRED_UPDATE_ON_SCHEME_BOARD; - } - - void DoExecute(TTransactionContext&, const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxPublishToSchemeBoard DoExecute" - << ", at schemeshard: " << Self->TabletID()); - - ui32 size = 0; - while (!Paths.empty() && size++ < Self->PublishChunkSize) { - auto it = Paths.begin(); - const auto txId = it->first; - auto& paths = it->second; - - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxPublishToSchemeBoard DescribePath" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId - << ", path id: " << paths.front()); - - Descriptions[txId].emplace_back(DescribePath(Self, ctx, paths.front())); - paths.pop_front(); - - if (paths.empty()) { - Paths.erase(it); - } - } - } - - void DoComplete(const TActorContext& ctx) override { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxPublishToSchemeBoard DoComplete" - << ", at schemeshard: " << Self->TabletID()); - - for (auto& kv : Descriptions) { - const auto txId = kv.first; - auto& descriptions = kv.second; - - for (auto& desc : descriptions) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxPublishToSchemeBoard Send" - << ", to populator: " << Self->SchemeBoardPopulator - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId - << ", path id: " << desc->Record.GetPathId()); - - ctx.Send(Self->SchemeBoardPopulator, std::move(desc), 0, ui64(txId)); - } - } - - if (Paths) { - Self->PublishToSchemeBoard(std::move(Paths), ctx); - } - } - -}; // TTxPublishToSchemeBoard - + + TTxPublishToSchemeBoard(TSelf *self, THashMap<TTxId, TDeque<TPathId>>&& paths) + : TRwTxBase(self) + , Paths(std::move(paths)) + { + } + + TTxType GetTxType() const override { + return TXTYPE_DEFERRED_UPDATE_ON_SCHEME_BOARD; + } + + void DoExecute(TTransactionContext&, const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPublishToSchemeBoard DoExecute" + << ", at schemeshard: " << Self->TabletID()); + + ui32 size = 0; + while (!Paths.empty() && size++ < Self->PublishChunkSize) { + auto it = Paths.begin(); + const auto txId = it->first; + auto& paths = it->second; + + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPublishToSchemeBoard DescribePath" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId + << ", path id: " << paths.front()); + + Descriptions[txId].emplace_back(DescribePath(Self, ctx, paths.front())); + paths.pop_front(); + + if (paths.empty()) { + Paths.erase(it); + } + } + } + + void DoComplete(const TActorContext& ctx) override { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPublishToSchemeBoard DoComplete" + << ", at schemeshard: " << Self->TabletID()); + + for (auto& kv : Descriptions) { + const auto txId = kv.first; + auto& descriptions = kv.second; + + for (auto& desc : descriptions) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPublishToSchemeBoard Send" + << ", to populator: " << Self->SchemeBoardPopulator + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId + << ", path id: " << desc->Record.GetPathId()); + + ctx.Send(Self->SchemeBoardPopulator, std::move(desc), 0, ui64(txId)); + } + } + + if (Paths) { + Self->PublishToSchemeBoard(std::move(Paths), ctx); + } + } + +}; // TTxPublishToSchemeBoard + struct TSchemeShard::TTxAckPublishToSchemeBoard: public TTransactionBase<TSchemeShard> { - TSchemeBoardEvents::TEvUpdateAck::TPtr Ev; - TSideEffects SideEffects; - - TTxAckPublishToSchemeBoard(TSelf *self, TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) - : TBase(self) - , Ev(ev) - { - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - const auto& record = Ev->Get()->Record; - - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxAckPublishToSchemeBoard Execute" - << ", at schemeshard: " << Self->TabletID() - << ", msg: " << record.ShortDebugString() - << ", cookie: " << Ev->Cookie); - + TSchemeBoardEvents::TEvUpdateAck::TPtr Ev; + TSideEffects SideEffects; + + TTxAckPublishToSchemeBoard(TSelf *self, TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) + : TBase(self) + , Ev(ev) + { + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + const auto& record = Ev->Get()->Record; + + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxAckPublishToSchemeBoard Execute" + << ", at schemeshard: " << Self->TabletID() + << ", msg: " << record.ShortDebugString() + << ", cookie: " << Ev->Cookie); + const auto txId = TTxId(Ev->Cookie); const auto pathId = TPathId(record.GetPathOwnerId(), record.GetLocalPathId()); - const ui64 version = record.GetVersion(); - - NIceDb::TNiceDb db(txc.DB); - - if (Self->Operations.contains(txId)) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Operation in-flight" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId); - - TOperation::TPtr operation = Self->Operations.at(txId); - if (AckPublish(db, txId, pathId, version, operation->Publications, ctx) - && operation->IsReadyToNotify(ctx)) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxAckPublishToSchemeBoard" - << ", operation is ready to notificate" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId); - - operation->DoNotify(Self, SideEffects, ctx); - } + const ui64 version = record.GetVersion(); + + NIceDb::TNiceDb db(txc.DB); + + if (Self->Operations.contains(txId)) { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Operation in-flight" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId); + + TOperation::TPtr operation = Self->Operations.at(txId); + if (AckPublish(db, txId, pathId, version, operation->Publications, ctx) + && operation->IsReadyToNotify(ctx)) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxAckPublishToSchemeBoard" + << ", operation is ready to notificate" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId); + + operation->DoNotify(Self, SideEffects, ctx); + } auto toActivateWaitPublication = operation->ActivatePartsWaitPublication(pathId, version); for (auto opId: toActivateWaitPublication) { @@ -137,99 +137,99 @@ struct TSchemeShard::TTxAckPublishToSchemeBoard: public TTransactionBase<TScheme operation->Parts[opId.GetSubTxId()]->HandleReply(personalEv, context); } - } else if (Self->Publications.contains(txId)) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Publication in-flight" + } else if (Self->Publications.contains(txId)) { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Publication in-flight" << ", count: " << Self->Publications.at(txId).Paths.size() - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId); - - auto& publication = Self->Publications.at(txId); - if (AckPublish(db, txId, pathId, version, publication.Paths, ctx)) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Publication complete, notify & remove" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId - << ", subscribers: " << publication.Subscribers.size()); - - Notify(txId, publication.Subscribers, ctx); - Self->Publications.erase(txId); - } - } else { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Unknown operation & publication" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId); - } - - SideEffects.ApplyOnExecute(Self, txc, ctx); - return true; - } - - void Complete(const TActorContext& ctx) override { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxAckPublishToSchemeBoard Complete" - << ", at schemeshard: " << Self->TabletID() - << ", cookie: " << Ev->Cookie); - - SideEffects.ApplyOnComplete(Self, ctx); - } - -private: - bool AckPublish( - NIceDb::TNiceDb& db, - TTxId txId, TPathId pathId, ui64 version, - TSet<std::pair<TPathId, ui64>>& paths, - const TActorContext& ctx - ) { - auto it = paths.lower_bound({pathId, 0}); - while (it != paths.end() && it->first == pathId && it->second <= version) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "AckPublish" - << ", at schemeshard: " << Self->TabletID() - << ", txId: " << txId - << ", pathId: " << pathId - << ", version: " << it->second); - - Self->PersistRemovePublishingPath(db, txId, pathId, it->second); - - auto eraseIt = it; - ++it; - paths.erase(eraseIt); - } - - return paths.empty(); - } - + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId); + + auto& publication = Self->Publications.at(txId); + if (AckPublish(db, txId, pathId, version, publication.Paths, ctx)) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Publication complete, notify & remove" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId + << ", subscribers: " << publication.Subscribers.size()); + + Notify(txId, publication.Subscribers, ctx); + Self->Publications.erase(txId); + } + } else { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Unknown operation & publication" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId); + } + + SideEffects.ApplyOnExecute(Self, txc, ctx); + return true; + } + + void Complete(const TActorContext& ctx) override { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxAckPublishToSchemeBoard Complete" + << ", at schemeshard: " << Self->TabletID() + << ", cookie: " << Ev->Cookie); + + SideEffects.ApplyOnComplete(Self, ctx); + } + +private: + bool AckPublish( + NIceDb::TNiceDb& db, + TTxId txId, TPathId pathId, ui64 version, + TSet<std::pair<TPathId, ui64>>& paths, + const TActorContext& ctx + ) { + auto it = paths.lower_bound({pathId, 0}); + while (it != paths.end() && it->first == pathId && it->second <= version) { + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "AckPublish" + << ", at schemeshard: " << Self->TabletID() + << ", txId: " << txId + << ", pathId: " << pathId + << ", version: " << it->second); + + Self->PersistRemovePublishingPath(db, txId, pathId, it->second); + + auto eraseIt = it; + ++it; + paths.erase(eraseIt); + } + + return paths.empty(); + } + void Notify(TTxId txId, const THashSet<TActorId>& subscribers, const TActorContext& ctx) { - for (const auto& subscriber : subscribers) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxAckPublishToSchemeBoard Notify" - << " send TEvNotifyTxCompletionResult" - << ", at schemeshard: " << Self->TabletID() - << ", to actorId: " << subscriber); - + for (const auto& subscriber : subscribers) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxAckPublishToSchemeBoard Notify" + << " send TEvNotifyTxCompletionResult" + << ", at schemeshard: " << Self->TabletID() + << ", to actorId: " << subscriber); + SideEffects.Send(subscriber, new TEvSchemeShard::TEvNotifyTxCompletionResult(ui64(txId)), ui64(txId)); - } - } - -}; // TTxAckPublishToSchemeBoard - + } + } + +}; // TTxAckPublishToSchemeBoard + NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxPublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths) { - return new TTxPublishToSchemeBoard(this, std::move(paths)); -} - + return new TTxPublishToSchemeBoard(this, std::move(paths)); +} + void TSchemeShard::PublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths, const TActorContext& ctx) { - Execute(CreateTxPublishToSchemeBoard(std::move(paths)), ctx); -} - + Execute(CreateTxPublishToSchemeBoard(std::move(paths)), ctx); +} + void TSchemeShard::PublishToSchemeBoard(TTxId txId, TDeque<TPathId>&& paths, const TActorContext& ctx) { - PublishToSchemeBoard({{txId, std::move(paths)}}, ctx); -} - + PublishToSchemeBoard({{txId, std::move(paths)}}, ctx); +} + NTabletFlatExecutor::ITransaction* TSchemeShard::CreateTxAckPublishToSchemeBoard(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev) { - return new TTxAckPublishToSchemeBoard(this, ev); -} - + return new TTxAckPublishToSchemeBoard(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard__serverless_storage_billing.cpp b/ydb/core/tx/schemeshard/schemeshard__serverless_storage_billing.cpp index cbc2c8d91e0..fdb4593a73e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__serverless_storage_billing.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__serverless_storage_billing.cpp @@ -31,11 +31,11 @@ struct TSchemeShard::TTxServerlessStorageBilling : public TTransactionBase<TSche const TSubDomainInfo::TPtr domainDescr = Self->SubDomains.at(Self->RootPathId()); const TSubDomainInfo::TDiskSpaceUsage& spaceUsage = domainDescr->GetDiskSpaceUsage(); - if (!Self->IsServerlessDomain(TPath::Init(Self->RootPathId(), Self))) { + if (!Self->IsServerlessDomain(TPath::Init(Self->RootPathId(), Self))) { LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxServerlessStorageBilling: unable to make a bill, domain is not a serverless db" << ", schemeshardId: " << Self->SelfTabletId() - << ", domainId: " << Self->ParentDomainId); + << ", domainId: " << Self->ParentDomainId); return true; } diff --git a/ydb/core/tx/schemeshard/schemeshard__state_changed_reply.cpp b/ydb/core/tx/schemeshard/schemeshard__state_changed_reply.cpp index 8a43af434bc..ffa6e34d509 100644 --- a/ydb/core/tx/schemeshard/schemeshard__state_changed_reply.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__state_changed_reply.cpp @@ -86,7 +86,7 @@ struct TSchemeShard::TTxShardStateChanged : public TSchemeShard::TRwTxBase { // Ack state change notification THolder<TEvDataShard::TEvStateChangedResult> event = THolder(new TEvDataShard::TEvStateChangedResult(Self->TabletID(), state)); - SideEffects.Send(Ev->Get()->GetSource(), std::move(event)); + SideEffects.Send(Ev->Get()->GetSource(), std::move(event)); if (state == NDataShard::TShardState::Offline) { DeleteShard(tabletId, ctx); diff --git a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp index 3532b6442f5..d04be460803 100644 --- a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp @@ -90,9 +90,9 @@ struct TSchemeShard::TTxUpdateTenant : public TSchemeShard::TRwTxBase { Self->PersistParentDomainEffectiveACL(db, record.GetOwner(), record.GetEffectiveACL(), record.GetEffectiveACLVersion()); for (const TPathId pathId : Self->ListSubThee(Self->RootPathId(), ctx)) { - SideEffects.PublishToSchemeBoard(InvalidOperationId, pathId); - } - + SideEffects.PublishToSchemeBoard(InvalidOperationId, pathId); + } + MakeSync(); } diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 093303ac123..b2e7db88b79 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -57,36 +57,36 @@ auto TSchemeShard::BuildStatsForCollector(TPathId pathId, TShardIdx shardIdx, TT sysStats.SetStartTime(*startTime); } - return ev; + return ev; } class TTxStorePartitionStats: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> { - TEvDataShard::TEvPeriodicTableStats::TPtr Ev; - - THolder<NSysView::TEvSysView::TEvSendPartitionStats> StatsCollectorEv; - THolder<TEvDataShard::TEvGetTableStats> GetStatsEv; + TEvDataShard::TEvPeriodicTableStats::TPtr Ev; + + THolder<NSysView::TEvSysView::TEvSendPartitionStats> StatsCollectorEv; + THolder<TEvDataShard::TEvGetTableStats> GetStatsEv; THolder<TEvDataShard::TEvCompactBorrowed> CompactEv; - + TSideEffects MergeOpSideEffects; -public: - explicit TTxStorePartitionStats(TSelf* self, TEvDataShard::TEvPeriodicTableStats::TPtr& ev) - : TBase(self) - , Ev(ev) - { - } - - virtual ~TTxStorePartitionStats() = default; - - TTxType GetTxType() const override { - return TXTYPE_STORE_PARTITION_STATS; - } - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - void Complete(const TActorContext& ctx) override; - -}; // TTxStorePartitionStats - +public: + explicit TTxStorePartitionStats(TSelf* self, TEvDataShard::TEvPeriodicTableStats::TPtr& ev) + : TBase(self) + , Ev(ev) + { + } + + virtual ~TTxStorePartitionStats() = default; + + TTxType GetTxType() const override { + return TXTYPE_STORE_PARTITION_STATS; + } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + void Complete(const TActorContext& ctx) override; + +}; // TTxStorePartitionStats + THolder<TProposeRequest> MergeRequest( TSchemeShard* ss, TTxId& txId, TPathId& pathId, const TVector<TShardIdx>& shardsToMerge) { @@ -114,15 +114,15 @@ THolder<TProposeRequest> MergeRequest( return std::move(request); } -bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorContext& ctx) { - const auto& rec = Ev->Get()->Record; +bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorContext& ctx) { + const auto& rec = Ev->Get()->Record; auto datashardId = TTabletId(rec.GetDatashardId()); TPathId tableId = InvalidPathId; if (rec.HasTableOwnerId()) { tableId = TPathId(TOwnerId(rec.GetTableOwnerId()), TLocalPathId(rec.GetTableLocalId())); } else { - tableId = Self->MakeLocalId(TLocalPathId(rec.GetTableLocalId())); + tableId = Self->MakeLocalId(TLocalPathId(rec.GetTableLocalId())); } const auto& tableStats = rec.GetTableStats(); @@ -130,17 +130,17 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte ui64 dataSize = tableStats.GetDataSize(); ui64 rowCount = tableStats.GetRowCount(); - if (!Self->Tables.contains(tableId)) { - return true; - } + if (!Self->Tables.contains(tableId)) { + return true; + } - TTableInfo::TPtr table = Self->Tables[tableId]; + TTableInfo::TPtr table = Self->Tables[tableId]; - if (!Self->TabletIdToShardIdx.contains(datashardId)) { - return true; - } + if (!Self->TabletIdToShardIdx.contains(datashardId)) { + return true; + } - auto shardIdx = Self->TabletIdToShardIdx[datashardId]; + auto shardIdx = Self->TabletIdToShardIdx[datashardId]; TTableInfo::TPartitionStats newStats; newStats.SeqNo = TMessageSeqNo(rec.GetGeneration(), rec.GetRound()); @@ -187,7 +187,7 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte } newStats.ShardState = rec.GetShardState(); - auto oldAggrStats = table->GetStats().Aggregated; + auto oldAggrStats = table->GetStats().Aggregated; table->UpdateShardStats(shardIdx, newStats); if (Self->CompactionQueue) { @@ -203,22 +203,22 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte NIceDb::TNiceDb db(txc.DB); if (!table->IsBackup && !table->IsShardsStatsDetached()) { - auto newAggrStats = table->GetStats().Aggregated; - auto subDomainId = Self->ResolveDomainId(tableId); - auto subDomainInfo = Self->ResolveDomainInfo(tableId); + auto newAggrStats = table->GetStats().Aggregated; + auto subDomainId = Self->ResolveDomainId(tableId); + auto subDomainInfo = Self->ResolveDomainInfo(tableId); subDomainInfo->AggrDiskSpaceUsage(Self, newAggrStats, oldAggrStats); if (subDomainInfo->CheckDiskSpaceQuotas(Self)) { - Self->PersistSubDomainState(db, subDomainId, *subDomainInfo); - // Publish is done in a separate transaction, so we may call this directly - TDeque<TPathId> toPublish; - toPublish.push_back(subDomainId); - Self->PublishToSchemeBoard(TTxId(), std::move(toPublish), ctx); - } + Self->PersistSubDomainState(db, subDomainId, *subDomainInfo); + // Publish is done in a separate transaction, so we may call this directly + TDeque<TPathId> toPublish; + toPublish.push_back(subDomainId); + Self->PublishToSchemeBoard(TTxId(), std::move(toPublish), ctx); + } } - - Self->PersistTablePartitionStats(db, tableId, shardIdx, table); - - if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) { + + Self->PersistTablePartitionStats(db, tableId, shardIdx, table); + + if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) { TMaybe<ui32> nodeId; if (rec.HasNodeId()) { nodeId = rec.GetNodeId(); @@ -227,34 +227,34 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte if (rec.HasStartTime()) { startTime = rec.GetStartTime(); } - StatsCollectorEv = Self->BuildStatsForCollector(tableId, shardIdx, datashardId, nodeId, startTime, newStats); + StatsCollectorEv = Self->BuildStatsForCollector(tableId, shardIdx, datashardId, nodeId, startTime, newStats); + } + + const auto& shardToPartition = table->GetShard2PartitionIdx(); + if (table->IsTTLEnabled() && shardToPartition.contains(shardIdx)) { + const ui64 partitionIdx = shardToPartition.at(shardIdx); + const auto& partitions = table->GetPartitions(); + + Y_VERIFY(partitionIdx < partitions.size()); + auto& shardInfo = partitions.at(partitionIdx); + auto& lag = shardInfo.LastCondEraseLag; + + if (lag) { + Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); + } + + if (shardInfo.LastCondErase) { + const auto now = ctx.Now(); + if (now >= shardInfo.LastCondErase) { + lag = now - shardInfo.LastCondErase; + } else { + lag = TDuration::Zero(); + } + + Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); + } } - const auto& shardToPartition = table->GetShard2PartitionIdx(); - if (table->IsTTLEnabled() && shardToPartition.contains(shardIdx)) { - const ui64 partitionIdx = shardToPartition.at(shardIdx); - const auto& partitions = table->GetPartitions(); - - Y_VERIFY(partitionIdx < partitions.size()); - auto& shardInfo = partitions.at(partitionIdx); - auto& lag = shardInfo.LastCondEraseLag; - - if (lag) { - Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); - } - - if (shardInfo.LastCondErase) { - const auto now = ctx.Now(); - if (now >= shardInfo.LastCondErase) { - lag = now - shardInfo.LastCondErase; - } else { - lag = TDuration::Zero(); - } - - Self->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); - } - } - TVector<TShardIdx> shardsToMerge; if (table->CheckCanMergePartitions(Self->SplitSettings, shardIdx, shardsToMerge)) { TTxId txId = Self->GetCachedTxId(ctx); @@ -279,34 +279,34 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte dbChanges.Apply(Self, txc, ctx); MergeOpSideEffects.ApplyOnExecute(Self, txc, ctx); - return true; + return true; } if (rec.GetShardState() != NKikimrTxDataShard::Ready) { - return true; + return true; } ui64 dataSizeResolution = 0; // Datashard will use default resolution ui64 rowCountResolution = 0; // Datashard will use default resolution bool collectKeySample = false; - if (table->CheckFastSplitForPartition(Self->SplitSettings, shardIdx, dataSize, rowCount)) { + if (table->CheckFastSplitForPartition(Self->SplitSettings, shardIdx, dataSize, rowCount)) { dataSizeResolution = Max<ui64>(dataSize / 100, 100*1024); rowCountResolution = Max<ui64>(rowCount / 100, 1000); collectKeySample = true; } else if (table->CheckSplitByLoad(Self->SplitSettings, shardIdx, dataSize, rowCount)) { collectKeySample = true; } else if (dataSize < table->GetShardSizeToSplit()) { - return true; + return true; } if (table->GetPartitions().size() >= table->GetMaxPartitionsCount()) { - return true; + return true; } { constexpr ui64 deltaShards = 2; - TPathElement::TPtr path = Self->PathsById.at(tableId); - TSubDomainInfo::TPtr domainInfo = Self->ResolveDomainInfo(tableId); + TPathElement::TPtr path = Self->PathsById.at(tableId); + TSubDomainInfo::TPtr domainInfo = Self->ResolveDomainInfo(tableId); if (domainInfo->GetShardsInside() + deltaShards > domainInfo->GetSchemeLimits().MaxShards) { LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -316,7 +316,7 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte << ", shards limit for domain: " << domainInfo->GetSchemeLimits().MaxShards << ", shards count inside domain: " << domainInfo->GetShardsInside() << ", intention to create new shards: " << deltaShards); - return true; + return true; } if (path->GetShardsInside() + deltaShards > domainInfo->GetSchemeLimits().MaxShardsInPath) { @@ -327,7 +327,7 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte << ", shards limit for path: " << domainInfo->GetSchemeLimits().MaxShardsInPath << ", shards count inside path: " << path->GetShardsInside() << ", intention to create new shards: " << deltaShards); - return true; + return true; } } @@ -338,55 +338,55 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte return true; } - // Request histograms from the datashard + // Request histograms from the datashard GetStatsEv.Reset(new TEvDataShard::TEvGetTableStats(tableId.LocalPathId, dataSizeResolution, rowCountResolution, collectKeySample)); - return true; + return true; } -void TTxStorePartitionStats::Complete(const TActorContext& ctx) { +void TTxStorePartitionStats::Complete(const TActorContext& ctx) { MergeOpSideEffects.ApplyOnComplete(Self, ctx); - if (StatsCollectorEv) { - ctx.Send(Self->SysPartitionStatsCollector, StatsCollectorEv.Release()); - } - + if (StatsCollectorEv) { + ctx.Send(Self->SysPartitionStatsCollector, StatsCollectorEv.Release()); + } + if (CompactEv) { LOG_DEBUG(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Requesting borrowed compaction from datasbard %" PRIu64, Ev->Get()->Record.GetDatashardId()); ctx.Send(Ev->Sender, CompactEv.Release()); } - if (GetStatsEv) { - LOG_DEBUG(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Requesting full stats from datashard %" PRIu64, Ev->Get()->Record.GetDatashardId()); - ctx.Send(Ev->Sender, GetStatsEv.Release()); - } -} - + if (GetStatsEv) { + LOG_DEBUG(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Requesting full stats from datashard %" PRIu64, Ev->Get()->Record.GetDatashardId()); + ctx.Send(Ev->Sender, GetStatsEv.Release()); + } +} + void TSchemeShard::Handle(TEvDataShard::TEvPeriodicTableStats::TPtr& ev, const TActorContext& ctx) { - const auto& rec = ev->Get()->Record; - - auto datashardId = TTabletId(rec.GetDatashardId()); - const auto& tableStats = rec.GetTableStats(); - const auto& tabletMetrics = rec.GetTabletMetrics(); - ui64 dataSize = tableStats.GetDataSize(); - ui64 rowCount = tableStats.GetRowCount(); - + const auto& rec = ev->Get()->Record; + + auto datashardId = TTabletId(rec.GetDatashardId()); + const auto& tableStats = rec.GetTableStats(); + const auto& tabletMetrics = rec.GetTabletMetrics(); + ui64 dataSize = tableStats.GetDataSize(); + ui64 rowCount = tableStats.GetRowCount(); + TPathId pathId = rec.HasTableOwnerId() ? TPathId(TOwnerId(rec.GetTableOwnerId()), TLocalPathId(rec.GetTableLocalId())) : MakeLocalId(TLocalPathId(rec.GetTableLocalId())); - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got periodic table stats at tablet " << TabletID() + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Got periodic table stats at tablet " << TabletID() << " from datashard " << datashardId << " pathId " << pathId - << " state '" << DatashardStateName(rec.GetShardState()) << "'" - << " dataSize " << dataSize - << " rowCount " << rowCount - << " cpuUsage " << tabletMetrics.GetCPU()/10000.0); - - Execute(new TTxStorePartitionStats(this, ev), ctx); -} - + << " state '" << DatashardStateName(rec.GetShardState()) << "'" + << " dataSize " << dataSize + << " rowCount " << rowCount + << " cpuUsage " << tabletMetrics.GetCPU()/10000.0); + + Execute(new TTxStorePartitionStats(this, ev), ctx); +} + }} diff --git a/ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp b/ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp index 0f426ac6c64..17049a628f2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp @@ -1,20 +1,20 @@ -#include "schemeshard_billing_helpers.h" - -#include <library/cpp/json/json_writer.h> - -#include <util/generic/size_literals.h> -#include <util/string/cast.h> - -namespace NKikimr { +#include "schemeshard_billing_helpers.h" + +#include <library/cpp/json/json_writer.h> + +#include <util/generic/size_literals.h> +#include <util/string/cast.h> + +namespace NKikimr { namespace NSchemeShard { - -ui64 TRUCalculator::ReadTable(ui64 bytes) { - return 128 * ((bytes + 1_MB - 1) / 1_MB); -} - -ui64 TRUCalculator::BulkUpsert(ui64 bytes, ui64 rows) { - return (Max(rows, (bytes + 1_KB - 1) / 1_KB) + 1) / 2; -} - + +ui64 TRUCalculator::ReadTable(ui64 bytes) { + return 128 * ((bytes + 1_MB - 1) / 1_MB); +} + +ui64 TRUCalculator::BulkUpsert(ui64 bytes, ui64 rows) { + return (Max(rows, (bytes + 1_KB - 1) / 1_KB) + 1) / 2; +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_billing_helpers.h b/ydb/core/tx/schemeshard/schemeshard_billing_helpers.h index 42fbaf3be47..5892f1e2b7a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_billing_helpers.h +++ b/ydb/core/tx/schemeshard/schemeshard_billing_helpers.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <ydb/core/metering/bill_record.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - -struct TRUCalculator { - // https://a.yandex-team.ru/arc/trunk/arcadia/kikimr/docs/ru/pricing/serverless.md - static ui64 ReadTable(ui64 bytes); - static ui64 BulkUpsert(ui64 bytes, ui64 rows); - -}; // TRUCalculator - + +struct TRUCalculator { + // https://a.yandex-team.ru/arc/trunk/arcadia/kikimr/docs/ru/pricing/serverless.md + static ui64 ReadTable(ui64 bytes); + static ui64 BulkUpsert(ui64 bytes, ui64 rows); + +}; // TRUCalculator + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp index 6ea21ab62f8..6d8cdc38dcc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp @@ -187,46 +187,46 @@ void TSchemeShard::PersistBuildIndexForget(NIceDb::TNiceDb& db, const TIndexBuil } void TSchemeShard::Resume(const TDeque<TIndexBuildId>& indexIds, const TActorContext& ctx) { - for (const auto& id : indexIds) { - if (IndexBuilds.contains(id)) { - Execute(CreateTxProgress(id), ctx); - } - } + for (const auto& id : indexIds) { + if (IndexBuilds.contains(id)) { + Execute(CreateTxProgress(id), ctx); + } + } } - + void TSchemeShard::SetupRouting(const TDeque<TIndexBuildId>& indexIds, const TActorContext &) { - for (const auto& id : indexIds) { - if (!IndexBuilds.contains(id)) { - continue; - } - - auto buildInfo = IndexBuilds.at(id); - - if (buildInfo->LockTxId) { - Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->LockTxId) - || TxIdToIndexBuilds.at(buildInfo->LockTxId) == buildInfo->Id); - TxIdToIndexBuilds[buildInfo->LockTxId] = buildInfo->Id; - } - - if (buildInfo->InitiateTxId) { - Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->InitiateTxId) - || TxIdToIndexBuilds.at(buildInfo->InitiateTxId) == buildInfo->Id); - TxIdToIndexBuilds[buildInfo->InitiateTxId] = buildInfo->Id; - } - - if (buildInfo->ApplyTxId) { - Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->ApplyTxId) - || TxIdToIndexBuilds.at(buildInfo->ApplyTxId) == buildInfo->Id); - TxIdToIndexBuilds[buildInfo->ApplyTxId] = buildInfo->Id; - } - - if (buildInfo->UnlockTxId) { - Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->UnlockTxId) - || TxIdToIndexBuilds.at(buildInfo->UnlockTxId) == buildInfo->Id); - TxIdToIndexBuilds[buildInfo->UnlockTxId] = buildInfo->Id; - } - } -} - -} -} + for (const auto& id : indexIds) { + if (!IndexBuilds.contains(id)) { + continue; + } + + auto buildInfo = IndexBuilds.at(id); + + if (buildInfo->LockTxId) { + Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->LockTxId) + || TxIdToIndexBuilds.at(buildInfo->LockTxId) == buildInfo->Id); + TxIdToIndexBuilds[buildInfo->LockTxId] = buildInfo->Id; + } + + if (buildInfo->InitiateTxId) { + Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->InitiateTxId) + || TxIdToIndexBuilds.at(buildInfo->InitiateTxId) == buildInfo->Id); + TxIdToIndexBuilds[buildInfo->InitiateTxId] = buildInfo->Id; + } + + if (buildInfo->ApplyTxId) { + Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->ApplyTxId) + || TxIdToIndexBuilds.at(buildInfo->ApplyTxId) == buildInfo->Id); + TxIdToIndexBuilds[buildInfo->ApplyTxId] = buildInfo->Id; + } + + if (buildInfo->UnlockTxId) { + Y_VERIFY(!TxIdToIndexBuilds.contains(buildInfo->UnlockTxId) + || TxIdToIndexBuilds.at(buildInfo->UnlockTxId) == buildInfo->Id); + TxIdToIndexBuilds[buildInfo->UnlockTxId] = buildInfo->Id; + } + } +} + +} +} diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp index 8a0bf65e2df..8a94bce2981 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp @@ -39,32 +39,32 @@ public: auto response = MakeHolder<TEvIndexBuilder::TEvCreateResponse>(request.GetTxId()); - switch (request.GetSettings().index().type_case()) { - case Ydb::Table::TableIndex::kGlobalIndex: - break; - case Ydb::Table::TableIndex::kGlobalAsyncIndex: - if (!Self->EnableAsyncIndexes) { + switch (request.GetSettings().index().type_case()) { + case Ydb::Table::TableIndex::kGlobalIndex: + break; + case Ydb::Table::TableIndex::kGlobalAsyncIndex: + if (!Self->EnableAsyncIndexes) { return Reply( std::move(response), Ydb::StatusIds::UNSUPPORTED, - TStringBuilder() << "Async indexes are not supported yet" + TStringBuilder() << "Async indexes are not supported yet" ); } - break; - default: - break; + break; + default: + break; } const auto& dataColumns = request.GetSettings().index().data_columns(); if (!dataColumns.empty() && !Self->AllowDataColumnForIndexTable) { - return Reply( - std::move(response), - Ydb::StatusIds::UNSUPPORTED, - TStringBuilder() << "Creating covered index is unsupported yet" - ); - } - + return Reply( + std::move(response), + Ydb::StatusIds::UNSUPPORTED, + TStringBuilder() << "Creating covered index is unsupported yet" + ); + } + const auto id = TIndexBuildId(request.GetTxId()); if (Self->IndexBuilds.contains(id)) { return Reply( @@ -192,9 +192,9 @@ private: case Ydb::Table::TableIndex::TypeCase::kGlobalIndex: buildInfo->IndexType = NKikimrSchemeOp::EIndexType::EIndexTypeGlobal; break; - case Ydb::Table::TableIndex::TypeCase::kGlobalAsyncIndex: + case Ydb::Table::TableIndex::TypeCase::kGlobalAsyncIndex: buildInfo->IndexType = NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync; - break; + break; case Ydb::Table::TableIndex::TypeCase::TYPE_NOT_SET: explain = "invalid or unset index type"; return false; @@ -237,7 +237,7 @@ private: AddIssue(record.MutableIssues(), errorMessage); } - Send(Request->Sender, std::move(responseEv), 0, Request->Cookie); + Send(Request->Sender, std::move(responseEv), 0, Request->Cookie); return true; } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index e8661ca0c3c..a5fbbae7037 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -172,7 +172,7 @@ public: case TIndexBuildInfo::EState::Locking: if (buildInfo->LockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->LockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), LockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->LockTxDone) { @@ -189,7 +189,7 @@ public: break; case TIndexBuildInfo::EState::Initiating: if (buildInfo->InitiateTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->InitiateTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), InitiatePropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->InitiateTxDone) { @@ -331,7 +331,7 @@ public: break; case TIndexBuildInfo::EState::Applying: if (buildInfo->ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), ApplyPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->ApplyTxDone) { @@ -343,7 +343,7 @@ public: break; case TIndexBuildInfo::EState::Unlocking: if (buildInfo->UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->UnlockTxDone) { @@ -360,7 +360,7 @@ public: case TIndexBuildInfo::EState::Cancellation_Applying: if (buildInfo->ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->ApplyTxDone) { @@ -372,7 +372,7 @@ public: break; case TIndexBuildInfo::EState::Cancellation_Unlocking: if (buildInfo->UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->UnlockTxDone) { @@ -389,7 +389,7 @@ public: case TIndexBuildInfo::EState::Rejection_Applying: if (buildInfo->ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->ApplyTxDone) { @@ -401,7 +401,7 @@ public: break; case TIndexBuildInfo::EState::Rejection_Unlocking: if (buildInfo->UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); + Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId)); } else if (buildInfo->UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo->UnlockTxDone) { @@ -925,7 +925,7 @@ public: << ", replyTo: " << buildInfo->CreateSender.ToString()); LOG_D("Message:\n" << responseEv->Record.ShortDebugString()); - Send(buildInfo->CreateSender, std::move(responseEv), 0, buildInfo->SenderCookie); + Send(buildInfo->CreateSender, std::move(responseEv), 0, buildInfo->SenderCookie); } bool OnModifyResult(TTransactionContext& txc, const TActorContext&) { diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h index 69db2c35196..f2e4d9e56bd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h @@ -2,7 +2,7 @@ #include "schemeshard_impl.h" #include "schemeshard_identificators.h" -#include "schemeshard_billing_helpers.h" +#include "schemeshard_billing_helpers.h" #include "schemeshard_build_index_helpers.h" #include "schemeshard__operation_part.h" // TSideEffects, make separate file @@ -62,8 +62,8 @@ private: } ui64 RequestUnits(const TBillingStats& stats) { - return TRUCalculator::ReadTable(stats.GetBytes()) - + TRUCalculator::BulkUpsert(stats.GetBytes(), stats.GetRows()); + return TRUCalculator::ReadTable(stats.GetBytes()) + + TRUCalculator::BulkUpsert(stats.GetBytes(), stats.GetRows()); } void RoundPeriod(TInstant& start, TInstant& end) { @@ -134,14 +134,14 @@ private: continue; } - if (!Self->IsServerlessDomain(domain)) { + if (!Self->IsServerlessDomain(domain)) { LOG_N("ApplyBill: unable to make a bill, domain is not a serverless db" << ", build index operation: " << buildId << ", domain: " << domain.PathString() << ", domainId: " << buildInfo->DomainPathId - << ", IsDomainSchemeShard: " << Self->IsDomainSchemeShard - << ", ParentDomainId: " << Self->ParentDomainId - << ", ResourcesDomainId: " << domain.DomainInfo()->GetResourcesDomainId() + << ", IsDomainSchemeShard: " << Self->IsDomainSchemeShard + << ", ParentDomainId: " << Self->ParentDomainId + << ", ResourcesDomainId: " << domain.DomainInfo()->GetResourcesDomainId() << ", not billed usage: " << toBill); continue; } @@ -159,14 +159,14 @@ private: << buildInfo->Billed.GetRows() << "-" << buildInfo->Billed.GetBytes() << "-" << buildInfo->Processed.GetRows() << "-" << buildInfo->Processed.GetBytes(); - const TString billRecord = TBillRecord() - .Id(id) - .CloudId(cloud_id) - .FolderId(folder_id) - .ResourceId(database_id) - .SourceWt(ctx.Now()) - .Usage(TBillRecord::RequestUnits(requestUnits, startPeriod, endPeriod)) - .ToString(); + const TString billRecord = TBillRecord() + .Id(id) + .CloudId(cloud_id) + .FolderId(folder_id) + .ResourceId(database_id) + .SourceWt(ctx.Now()) + .Usage(TBillRecord::RequestUnits(requestUnits, startPeriod, endPeriod)) + .ToString(); LOG_D("ApplyBill: made a bill" << ", buildInfo: " << *buildInfo @@ -179,7 +179,7 @@ private: } protected: - void Send(TActorId dst, THolder<IEventBase> message, ui32 flags = 0, ui64 cookie = 0) { + void Send(TActorId dst, THolder<IEventBase> message, ui32 flags = 0, ui64 cookie = 0) { SideEffects.Send(dst, message.Release(), cookie, flags); } @@ -282,7 +282,7 @@ protected: break; case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; + break; case NKikimrSchemeOp::EIndexType::EIndexTypeInvalid: Y_FAIL("Unreachable"); }; diff --git a/ydb/core/tx/schemeshard/schemeshard_export.cpp b/ydb/core/tx/schemeshard/schemeshard_export.cpp index 64d2fc09a76..00eef5a0dbc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export.cpp @@ -1,223 +1,223 @@ -#include "schemeshard_export.h" -#include "schemeshard_export_flow_proposals.h" -#include "schemeshard_impl.h" - -#include <util/generic/xrange.h> - -namespace NKikimr { +#include "schemeshard_export.h" +#include "schemeshard_export_flow_proposals.h" +#include "schemeshard_impl.h" + +#include <util/generic/xrange.h> + +namespace NKikimr { namespace NSchemeShard { - -namespace { - - void AddIssue(NKikimrExport::TExport& exprt, const TString& errorMessage) { - auto& issue = *exprt.AddIssues(); - issue.set_severity(NYql::TSeverityIds::S_ERROR); - issue.set_message(errorMessage); - } - - void FillIssues(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo) { - if (exportInfo->Issue) { - AddIssue(exprt, exportInfo->Issue); - } - - for (const auto& item : exportInfo->Items) { - if (item.Issue) { - AddIssue(exprt, item.Issue); - } - } - } - - NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) { - NProtoBuf::Timestamp timestamp; - timestamp.set_seconds((i64)(sec)); - timestamp.set_nanos(0); - return timestamp; - } - + +namespace { + + void AddIssue(NKikimrExport::TExport& exprt, const TString& errorMessage) { + auto& issue = *exprt.AddIssues(); + issue.set_severity(NYql::TSeverityIds::S_ERROR); + issue.set_message(errorMessage); + } + + void FillIssues(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo) { + if (exportInfo->Issue) { + AddIssue(exprt, exportInfo->Issue); + } + + for (const auto& item : exportInfo->Items) { + if (item.Issue) { + AddIssue(exprt, item.Issue); + } + } + } + + NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) { + NProtoBuf::Timestamp timestamp; + timestamp.set_seconds((i64)(sec)); + timestamp.set_nanos(0); + return timestamp; + } + void FillItemProgress(TSchemeShard* ss, const TExportInfo::TPtr exportInfo, ui32 itemIdx, - Ydb::Export::ExportItemProgress& itemProgress) { - - Y_VERIFY(itemIdx < exportInfo->Items.size()); - - const auto& item = exportInfo->Items.at(itemIdx); - if (item.WaitTxId == InvalidTxId) { - return; - } - - const TOperationId opId(item.WaitTxId, FirstSubTxId); - const TPath path = TPath::Resolve(ExportItemPathName(ss, exportInfo, itemIdx), ss); - - if (ss->TxInFlight.contains(opId)) { - const auto& txState = ss->TxInFlight.at(opId); - if (txState.TxType != TTxState::TxBackup) { - return; - } - - itemProgress.set_parts_total(txState.Shards.size()); - itemProgress.set_parts_completed(txState.Shards.size() - txState.ShardsInProgress.size()); - *itemProgress.mutable_start_time() = SecondsToProtoTimeStamp(txState.StartTime.Seconds()); - } else if (path.IsResolved()) { - if (!ss->Tables.contains(path.Base()->PathId)) { - return; - } - - auto table = ss->Tables.at(path.Base()->PathId); - if (!table->BackupHistory.contains(item.WaitTxId)) { - return; - } - - const auto& backupResult = table->BackupHistory.at(item.WaitTxId); - itemProgress.set_parts_total(backupResult.TotalShardCount); - itemProgress.set_parts_completed(backupResult.TotalShardCount); - *itemProgress.mutable_start_time() = SecondsToProtoTimeStamp(backupResult.StartDateTime); - *itemProgress.mutable_end_time() = SecondsToProtoTimeStamp(backupResult.CompletionDateTime); - } - } - -} // anonymous - + Ydb::Export::ExportItemProgress& itemProgress) { + + Y_VERIFY(itemIdx < exportInfo->Items.size()); + + const auto& item = exportInfo->Items.at(itemIdx); + if (item.WaitTxId == InvalidTxId) { + return; + } + + const TOperationId opId(item.WaitTxId, FirstSubTxId); + const TPath path = TPath::Resolve(ExportItemPathName(ss, exportInfo, itemIdx), ss); + + if (ss->TxInFlight.contains(opId)) { + const auto& txState = ss->TxInFlight.at(opId); + if (txState.TxType != TTxState::TxBackup) { + return; + } + + itemProgress.set_parts_total(txState.Shards.size()); + itemProgress.set_parts_completed(txState.Shards.size() - txState.ShardsInProgress.size()); + *itemProgress.mutable_start_time() = SecondsToProtoTimeStamp(txState.StartTime.Seconds()); + } else if (path.IsResolved()) { + if (!ss->Tables.contains(path.Base()->PathId)) { + return; + } + + auto table = ss->Tables.at(path.Base()->PathId); + if (!table->BackupHistory.contains(item.WaitTxId)) { + return; + } + + const auto& backupResult = table->BackupHistory.at(item.WaitTxId); + itemProgress.set_parts_total(backupResult.TotalShardCount); + itemProgress.set_parts_completed(backupResult.TotalShardCount); + *itemProgress.mutable_start_time() = SecondsToProtoTimeStamp(backupResult.StartDateTime); + *itemProgress.mutable_end_time() = SecondsToProtoTimeStamp(backupResult.CompletionDateTime); + } + } + +} // anonymous + void TSchemeShard::FromXxportInfo(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo) { - exprt.SetId(exportInfo->Id); - exprt.SetStatus(Ydb::StatusIds::SUCCESS); - - switch (exportInfo->State) { - case TExportInfo::EState::CreateExportDir: - case TExportInfo::EState::CopyTables: - exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_PREPARING); - break; - - case TExportInfo::EState::Transferring: - case TExportInfo::EState::Done: - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - FillItemProgress(this, exportInfo, itemIdx, *exprt.AddItemsProgress()); - } - exprt.SetProgress(exportInfo->IsDone() - ? Ydb::Export::ExportProgress::PROGRESS_DONE - : Ydb::Export::ExportProgress::PROGRESS_TRANSFER_DATA); - break; - - case TExportInfo::EState::Dropping: - exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_DONE); - break; - - case TExportInfo::EState::Cancellation: - FillIssues(exprt, exportInfo); - exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_CANCELLATION); - break; - - case TExportInfo::EState::Cancelled: - exprt.SetStatus(Ydb::StatusIds::CANCELLED); - FillIssues(exprt, exportInfo); - exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_CANCELLED); - break; - - default: - exprt.SetStatus(Ydb::StatusIds::UNDETERMINED); - exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_UNSPECIFIED); - break; - } - - switch (exportInfo->Kind) { - case TExportInfo::EKind::YT: - Y_VERIFY(exprt.MutableExportToYtSettings()->ParseFromString(exportInfo->Settings)); - exprt.MutableExportToYtSettings()->clear_token(); - break; - - case TExportInfo::EKind::S3: - Y_VERIFY(exprt.MutableExportToS3Settings()->ParseFromString(exportInfo->Settings)); - exprt.MutableExportToS3Settings()->clear_access_key(); - exprt.MutableExportToS3Settings()->clear_secret_key(); - break; - } -} - + exprt.SetId(exportInfo->Id); + exprt.SetStatus(Ydb::StatusIds::SUCCESS); + + switch (exportInfo->State) { + case TExportInfo::EState::CreateExportDir: + case TExportInfo::EState::CopyTables: + exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_PREPARING); + break; + + case TExportInfo::EState::Transferring: + case TExportInfo::EState::Done: + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + FillItemProgress(this, exportInfo, itemIdx, *exprt.AddItemsProgress()); + } + exprt.SetProgress(exportInfo->IsDone() + ? Ydb::Export::ExportProgress::PROGRESS_DONE + : Ydb::Export::ExportProgress::PROGRESS_TRANSFER_DATA); + break; + + case TExportInfo::EState::Dropping: + exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_DONE); + break; + + case TExportInfo::EState::Cancellation: + FillIssues(exprt, exportInfo); + exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_CANCELLATION); + break; + + case TExportInfo::EState::Cancelled: + exprt.SetStatus(Ydb::StatusIds::CANCELLED); + FillIssues(exprt, exportInfo); + exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_CANCELLED); + break; + + default: + exprt.SetStatus(Ydb::StatusIds::UNDETERMINED); + exprt.SetProgress(Ydb::Export::ExportProgress::PROGRESS_UNSPECIFIED); + break; + } + + switch (exportInfo->Kind) { + case TExportInfo::EKind::YT: + Y_VERIFY(exprt.MutableExportToYtSettings()->ParseFromString(exportInfo->Settings)); + exprt.MutableExportToYtSettings()->clear_token(); + break; + + case TExportInfo::EKind::S3: + Y_VERIFY(exprt.MutableExportToS3Settings()->ParseFromString(exportInfo->Settings)); + exprt.MutableExportToS3Settings()->clear_access_key(); + exprt.MutableExportToS3Settings()->clear_secret_key(); + break; + } +} + void TSchemeShard::PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo) { - db.Table<Schema::Exports>().Key(exportInfo->Id).Update( - NIceDb::TUpdate<Schema::Exports::Uid>(exportInfo->Uid), - NIceDb::TUpdate<Schema::Exports::Kind>(static_cast<ui8>(exportInfo->Kind)), - NIceDb::TUpdate<Schema::Exports::Settings>(exportInfo->Settings), + db.Table<Schema::Exports>().Key(exportInfo->Id).Update( + NIceDb::TUpdate<Schema::Exports::Uid>(exportInfo->Uid), + NIceDb::TUpdate<Schema::Exports::Kind>(static_cast<ui8>(exportInfo->Kind)), + NIceDb::TUpdate<Schema::Exports::Settings>(exportInfo->Settings), NIceDb::TUpdate<Schema::Exports::DomainPathOwnerId>(exportInfo->DomainPathId.OwnerId), NIceDb::TUpdate<Schema::Exports::DomainPathId>(exportInfo->DomainPathId.LocalPathId), - NIceDb::TUpdate<Schema::Exports::Items>(exportInfo->Items.size()) - ); - - if (exportInfo->UserSID) { - db.Table<Schema::Exports>().Key(exportInfo->Id).Update( - NIceDb::TUpdate<Schema::Exports::UserSID>(*exportInfo->UserSID) - ); - } - - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - const auto& item = exportInfo->Items.at(itemIdx); - - db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ExportItems::SourcePathName>(item.SourcePathName), + NIceDb::TUpdate<Schema::Exports::Items>(exportInfo->Items.size()) + ); + + if (exportInfo->UserSID) { + db.Table<Schema::Exports>().Key(exportInfo->Id).Update( + NIceDb::TUpdate<Schema::Exports::UserSID>(*exportInfo->UserSID) + ); + } + + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + const auto& item = exportInfo->Items.at(itemIdx); + + db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ExportItems::SourcePathName>(item.SourcePathName), NIceDb::TUpdate<Schema::ExportItems::SourceOwnerPathId>(item.SourcePathId.OwnerId), NIceDb::TUpdate<Schema::ExportItems::SourcePathId>(item.SourcePathId.LocalPathId), - NIceDb::TUpdate<Schema::ExportItems::State>(static_cast<ui8>(item.State)) - ); - } -} - + NIceDb::TUpdate<Schema::ExportItems::State>(static_cast<ui8>(item.State)) + ); + } +} + void TSchemeShard::PersistRemoveExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo) { - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Delete(); - } - - db.Table<Schema::Exports>().Key(exportInfo->Id).Delete(); -} - + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Delete(); + } + + db.Table<Schema::Exports>().Key(exportInfo->Id).Delete(); +} + void TSchemeShard::PersistExportPathId(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo) { - db.Table<Schema::Exports>().Key(exportInfo->Id).Update( + db.Table<Schema::Exports>().Key(exportInfo->Id).Update( NIceDb::TUpdate<Schema::Exports::ExportOwnerPathId>(exportInfo->ExportPathId.OwnerId), NIceDb::TUpdate<Schema::Exports::ExportPathId>(exportInfo->ExportPathId.LocalPathId) - ); -} - + ); +} + void TSchemeShard::PersistExportState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo) { - db.Table<Schema::Exports>().Key(exportInfo->Id).Update( - NIceDb::TUpdate<Schema::Exports::State>(static_cast<ui8>(exportInfo->State)), - NIceDb::TUpdate<Schema::Exports::WaitTxId>(exportInfo->WaitTxId), - NIceDb::TUpdate<Schema::Exports::Issue>(exportInfo->Issue) - ); -} - + db.Table<Schema::Exports>().Key(exportInfo->Id).Update( + NIceDb::TUpdate<Schema::Exports::State>(static_cast<ui8>(exportInfo->State)), + NIceDb::TUpdate<Schema::Exports::WaitTxId>(exportInfo->WaitTxId), + NIceDb::TUpdate<Schema::Exports::Issue>(exportInfo->Issue) + ); +} + void TSchemeShard::PersistExportItemState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - const auto& item = exportInfo->Items.at(itemIdx); - - db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ExportItems::State>(static_cast<ui8>(item.State)), - NIceDb::TUpdate<Schema::ExportItems::BackupTxId>(item.WaitTxId), - NIceDb::TUpdate<Schema::ExportItems::Issue>(item.Issue) - ); -} - + Y_VERIFY(itemIdx < exportInfo->Items.size()); + const auto& item = exportInfo->Items.at(itemIdx); + + db.Table<Schema::ExportItems>().Key(exportInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ExportItems::State>(static_cast<ui8>(item.State)), + NIceDb::TUpdate<Schema::ExportItems::BackupTxId>(item.WaitTxId), + NIceDb::TUpdate<Schema::ExportItems::Issue>(item.Issue) + ); +} + void TSchemeShard::Handle(TEvExport::TEvCreateExportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxCreateExport(ev), ctx); -} - + Execute(CreateTxCreateExport(ev), ctx); +} + void TSchemeShard::Handle(TEvExport::TEvGetExportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxGetExport(ev), ctx); -} - + Execute(CreateTxGetExport(ev), ctx); +} + void TSchemeShard::Handle(TEvExport::TEvCancelExportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxCancelExport(ev), ctx); -} - + Execute(CreateTxCancelExport(ev), ctx); +} + void TSchemeShard::Handle(TEvExport::TEvForgetExportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxForgetExport(ev), ctx); -} - + Execute(CreateTxForgetExport(ev), ctx); +} + void TSchemeShard::Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxListExports(ev), ctx); -} - + Execute(CreateTxListExports(ev), ctx); +} + void TSchemeShard::ResumeExports(const TVector<ui64>& exportIds, const TActorContext& ctx) { - for (const ui64 id : exportIds) { - Execute(CreateTxProgressExport(id), ctx); - } -} - + for (const ui64 id : exportIds) { + Execute(CreateTxProgressExport(id), ctx); + } +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export.h b/ydb/core/tx/schemeshard/schemeshard_export.h index 9ac5bb149a1..b4c39a2128f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export.h +++ b/ydb/core/tx/schemeshard/schemeshard_export.h @@ -1,171 +1,171 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/protos/export.pb.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - -struct TEvExport { - enum EEv { - EvCreateExportRequest = EventSpaceBegin(TKikimrEvents::ES_EXPORT_SERVICE), - EvCreateExportResponse, - EvGetExportRequest, - EvGetExportResponse, - EvCancelExportRequest, - EvCancelExportResponse, - EvForgetExportRequest, - EvForgetExportResponse, - EvListExportsRequest, - EvListExportsResponse, - - EvEnd - }; - - static_assert( - EvEnd < EventSpaceEnd(TKikimrEvents::ES_EXPORT_SERVICE), - "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_EXPORT_SERVICE)" - ); - -#ifdef DECLARE_EVENT_CLASS -#error DECLARE_EVENT_CLASS macro redefinition -#else -#define DECLARE_EVENT_CLASS(NAME) struct T##NAME: public TEventPB<T##NAME, NKikimrExport::T##NAME, NAME> -#endif - - DECLARE_EVENT_CLASS(EvCreateExportRequest) { - TEvCreateExportRequest() = default; - - explicit TEvCreateExportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrExport::TCreateExportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - }; - - DECLARE_EVENT_CLASS(EvCreateExportResponse) { - TEvCreateExportResponse() = default; - - explicit TEvCreateExportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvGetExportRequest) { - TEvGetExportRequest() = default; - - explicit TEvGetExportRequest(const TString& dbName, const NKikimrExport::TGetExportRequest& request) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvGetExportRequest(const TString& dbName, const ui64 exportId) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(exportId); - } - }; - - DECLARE_EVENT_CLASS(EvGetExportResponse) { - }; - - DECLARE_EVENT_CLASS(EvCancelExportRequest) { - TEvCancelExportRequest() = default; - - explicit TEvCancelExportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrExport::TCancelExportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvCancelExportRequest( - const ui64 txId, - const TString& dbName, - const ui64 exportId - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(exportId); - } - }; - - DECLARE_EVENT_CLASS(EvCancelExportResponse) { - TEvCancelExportResponse() = default; - - explicit TEvCancelExportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvForgetExportRequest) { - TEvForgetExportRequest() = default; - - explicit TEvForgetExportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrExport::TForgetExportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvForgetExportRequest( - const ui64 txId, - const TString& dbName, - const ui64 exportId - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(exportId); - } - }; - - DECLARE_EVENT_CLASS(EvForgetExportResponse) { - TEvForgetExportResponse() = default; - - explicit TEvForgetExportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvListExportsRequest) { - TEvListExportsRequest() = default; - - explicit TEvListExportsRequest(const TString& dbName, const NKikimrExport::TListExportsRequest& request) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvListExportsRequest( - const TString& dbName, - const ui64 pageSize, - const TString& pageToken, - const TString& kind - ) { - Record.SetDatabaseName(dbName); - - auto& request = *Record.MutableRequest(); - request.SetPageSize(pageSize); - request.SetPageToken(pageToken); - request.SetKind(kind); - } - }; - - DECLARE_EVENT_CLASS(EvListExportsResponse) { - }; - -#undef DECLARE_EVENT_CLASS - -}; // TEvExport - + +struct TEvExport { + enum EEv { + EvCreateExportRequest = EventSpaceBegin(TKikimrEvents::ES_EXPORT_SERVICE), + EvCreateExportResponse, + EvGetExportRequest, + EvGetExportResponse, + EvCancelExportRequest, + EvCancelExportResponse, + EvForgetExportRequest, + EvForgetExportResponse, + EvListExportsRequest, + EvListExportsResponse, + + EvEnd + }; + + static_assert( + EvEnd < EventSpaceEnd(TKikimrEvents::ES_EXPORT_SERVICE), + "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_EXPORT_SERVICE)" + ); + +#ifdef DECLARE_EVENT_CLASS +#error DECLARE_EVENT_CLASS macro redefinition +#else +#define DECLARE_EVENT_CLASS(NAME) struct T##NAME: public TEventPB<T##NAME, NKikimrExport::T##NAME, NAME> +#endif + + DECLARE_EVENT_CLASS(EvCreateExportRequest) { + TEvCreateExportRequest() = default; + + explicit TEvCreateExportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrExport::TCreateExportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + }; + + DECLARE_EVENT_CLASS(EvCreateExportResponse) { + TEvCreateExportResponse() = default; + + explicit TEvCreateExportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvGetExportRequest) { + TEvGetExportRequest() = default; + + explicit TEvGetExportRequest(const TString& dbName, const NKikimrExport::TGetExportRequest& request) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvGetExportRequest(const TString& dbName, const ui64 exportId) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(exportId); + } + }; + + DECLARE_EVENT_CLASS(EvGetExportResponse) { + }; + + DECLARE_EVENT_CLASS(EvCancelExportRequest) { + TEvCancelExportRequest() = default; + + explicit TEvCancelExportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrExport::TCancelExportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvCancelExportRequest( + const ui64 txId, + const TString& dbName, + const ui64 exportId + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(exportId); + } + }; + + DECLARE_EVENT_CLASS(EvCancelExportResponse) { + TEvCancelExportResponse() = default; + + explicit TEvCancelExportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvForgetExportRequest) { + TEvForgetExportRequest() = default; + + explicit TEvForgetExportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrExport::TForgetExportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvForgetExportRequest( + const ui64 txId, + const TString& dbName, + const ui64 exportId + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(exportId); + } + }; + + DECLARE_EVENT_CLASS(EvForgetExportResponse) { + TEvForgetExportResponse() = default; + + explicit TEvForgetExportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvListExportsRequest) { + TEvListExportsRequest() = default; + + explicit TEvListExportsRequest(const TString& dbName, const NKikimrExport::TListExportsRequest& request) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvListExportsRequest( + const TString& dbName, + const ui64 pageSize, + const TString& pageToken, + const TString& kind + ) { + Record.SetDatabaseName(dbName); + + auto& request = *Record.MutableRequest(); + request.SetPageSize(pageSize); + request.SetPageToken(pageToken); + request.SetKind(kind); + } + }; + + DECLARE_EVENT_CLASS(EvListExportsResponse) { + }; + +#undef DECLARE_EVENT_CLASS + +}; // TEvExport + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export__cancel.cpp b/ydb/core/tx/schemeshard/schemeshard_export__cancel.cpp index fa75718f115..2282dd42bbb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__cancel.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__cancel.cpp @@ -1,182 +1,182 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_export_flow_proposals.h" -#include "schemeshard_export.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_export_flow_proposals.h" +#include "schemeshard_export.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/ptr.h> -#include <util/generic/xrange.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> +#include <util/generic/xrange.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TExport::TTxCancel: public TSchemeShard::TXxport::TTxBase { - TEvExport::TEvCancelExportRequest::TPtr Request; - - explicit TTxCancel(TSelf *self, TEvExport::TEvCancelExportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CANCEL_EXPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - auto response = MakeHolder<TEvExport::TEvCancelExportResponse>(request.GetTxId()); - auto& cancel = *response->Record.MutableResponse(); - - auto it = Self->Exports.find(request.GetRequest().GetId()); - if (it == Self->Exports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { - cancel.SetStatus(Ydb::StatusIds::NOT_FOUND); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - TExportInfo::TPtr exportInfo = it->second; - - if (!exportInfo->IsValid()) { - cancel.SetStatus(Ydb::StatusIds::UNDETERMINED); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - cancel.SetStatus(Ydb::StatusIds::SUCCESS); - - if (exportInfo->IsFinished() || exportInfo->IsDropping()) { - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - exportInfo->Issue = "Cancelled manually"; - - if (exportInfo->State < TExportInfo::EState::Transferring) { - exportInfo->State = TExportInfo::EState::Cancelled; - - Self->TxIdToExport.erase(exportInfo->WaitTxId); - exportInfo->WaitTxId = InvalidTxId; - } else { - exportInfo->State = TExportInfo::EState::Cancelled; - - for (const auto& item : exportInfo->Items) { - if (item.State != TExportInfo::EState::Transferring) { - continue; - } - - exportInfo->State = TExportInfo::EState::Cancellation; - if (item.WaitTxId != InvalidTxId) { - Send(Self->SelfId(), CancelPropose(exportInfo, item.WaitTxId), 0, exportInfo->Id); - } - } - } - - NIceDb::TNiceDb db(txc.DB); - Self->PersistExportState(db, exportInfo); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); + TEvExport::TEvCancelExportRequest::TPtr Request; + + explicit TTxCancel(TSelf *self, TEvExport::TEvCancelExportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CANCEL_EXPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + auto response = MakeHolder<TEvExport::TEvCancelExportResponse>(request.GetTxId()); + auto& cancel = *response->Record.MutableResponse(); + + auto it = Self->Exports.find(request.GetRequest().GetId()); + if (it == Self->Exports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { + cancel.SetStatus(Ydb::StatusIds::NOT_FOUND); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + TExportInfo::TPtr exportInfo = it->second; + + if (!exportInfo->IsValid()) { + cancel.SetStatus(Ydb::StatusIds::UNDETERMINED); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + cancel.SetStatus(Ydb::StatusIds::SUCCESS); + + if (exportInfo->IsFinished() || exportInfo->IsDropping()) { + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + exportInfo->Issue = "Cancelled manually"; + + if (exportInfo->State < TExportInfo::EState::Transferring) { + exportInfo->State = TExportInfo::EState::Cancelled; + + Self->TxIdToExport.erase(exportInfo->WaitTxId); + exportInfo->WaitTxId = InvalidTxId; + } else { + exportInfo->State = TExportInfo::EState::Cancelled; + + for (const auto& item : exportInfo->Items) { + if (item.State != TExportInfo::EState::Transferring) { + continue; + } + + exportInfo->State = TExportInfo::EState::Cancellation; + if (item.WaitTxId != InvalidTxId) { + Send(Self->SelfId(), CancelPropose(exportInfo, item.WaitTxId), 0, exportInfo->Id); + } + } + } + + NIceDb::TNiceDb db(txc.DB); + Self->PersistExportState(db, exportInfo); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); SendNotificationsIfFinished(exportInfo); - return true; - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxCancel - + return true; + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxCancel + struct TSchemeShard::TExport::TTxCancelAck: public TSchemeShard::TXxport::TTxBase { TEvSchemeShard::TEvCancelTxResult::TPtr CancelResult; - + explicit TTxCancelAck(TSelf *self, TEvSchemeShard::TEvCancelTxResult::TPtr& ev) - : TXxport::TTxBase(self) - , CancelResult(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CANCEL_EXPORT_ACK; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const ui64 id = CancelResult->Cookie; - const auto backupTxId = TTxId(CancelResult->Get()->Record.GetTargetTxId()); - - if (!Self->Exports.contains(id)) { - return true; - } - - TExportInfo::TPtr exportInfo = Self->Exports.at(id); - - if (exportInfo->State != TExportInfo::EState::Cancellation) { - return true; - } - - bool found = false; - ui32 itemIdx; - ui32 cancelledItems = 0; - ui32 cancellableItems = 0; - - for (ui32 i : xrange(exportInfo->Items.size())) { - auto& item = exportInfo->Items.at(i); - - if (item.State != TExportInfo::EState::Transferring) { - continue; - } - - if (item.WaitTxId != InvalidTxId) { - ++cancellableItems; - } - - if (item.WaitTxId == backupTxId) { - found = true; - - item.State = TExportInfo::EState::Cancelled; - itemIdx = i; - } - - if (item.State == TExportInfo::EState::Cancelled) { - ++cancelledItems; - } - } - - if (!found) { - return true; - } - - Self->TxIdToExport.erase(backupTxId); - - NIceDb::TNiceDb db(txc.DB); - Self->PersistExportItemState(db, exportInfo, itemIdx); - - if (cancelledItems == cancellableItems) { - exportInfo->State = TExportInfo::EState::Cancelled; - Self->PersistExportState(db, exportInfo); - } - + : TXxport::TTxBase(self) + , CancelResult(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CANCEL_EXPORT_ACK; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const ui64 id = CancelResult->Cookie; + const auto backupTxId = TTxId(CancelResult->Get()->Record.GetTargetTxId()); + + if (!Self->Exports.contains(id)) { + return true; + } + + TExportInfo::TPtr exportInfo = Self->Exports.at(id); + + if (exportInfo->State != TExportInfo::EState::Cancellation) { + return true; + } + + bool found = false; + ui32 itemIdx; + ui32 cancelledItems = 0; + ui32 cancellableItems = 0; + + for (ui32 i : xrange(exportInfo->Items.size())) { + auto& item = exportInfo->Items.at(i); + + if (item.State != TExportInfo::EState::Transferring) { + continue; + } + + if (item.WaitTxId != InvalidTxId) { + ++cancellableItems; + } + + if (item.WaitTxId == backupTxId) { + found = true; + + item.State = TExportInfo::EState::Cancelled; + itemIdx = i; + } + + if (item.State == TExportInfo::EState::Cancelled) { + ++cancelledItems; + } + } + + if (!found) { + return true; + } + + Self->TxIdToExport.erase(backupTxId); + + NIceDb::TNiceDb db(txc.DB); + Self->PersistExportItemState(db, exportInfo, itemIdx); + + if (cancelledItems == cancellableItems) { + exportInfo->State = TExportInfo::EState::Cancelled; + Self->PersistExportState(db, exportInfo); + } + SendNotificationsIfFinished(exportInfo); - return true; - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxCancelAck - + return true; + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxCancelAck + ITransaction* TSchemeShard::CreateTxCancelExport(TEvExport::TEvCancelExportRequest::TPtr& ev) { - return new TExport::TTxCancel(this, ev); -} - + return new TExport::TTxCancel(this, ev); +} + ITransaction* TSchemeShard::CreateTxCancelExportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev) { - return new TExport::TTxCancelAck(this, ev); -} - + return new TExport::TTxCancelAck(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index 7d03055a6bb..c297a9ea0b4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -1,954 +1,954 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_export_flow_proposals.h" -#include "schemeshard_export_helpers.h" -#include "schemeshard_export.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_export_flow_proposals.h" +#include "schemeshard_export_helpers.h" +#include "schemeshard_export.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_export.pb.h> #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/algorithm.h> -#include <util/generic/ptr.h> -#include <util/generic/xrange.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/algorithm.h> +#include <util/generic/ptr.h> +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase { - TEvExport::TEvCreateExportRequest::TPtr Request; - bool Progress; - - explicit TTxCreate(TSelf* self, TEvExport::TEvCreateExportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - , Progress(false) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CREATE_EXPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - LOG_D("TExport::TTxCreate: DoExecute"); - LOG_T("Message:\n" << request.ShortDebugString()); - - auto response = MakeHolder<TEvExport::TEvCreateExportResponse>(request.GetTxId()); - - const ui64 id = request.GetTxId(); - if (Self->Exports.contains(id)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Export with id '" << id << "' already exists" - ); - } - - const TString& uid = GetUid(request.GetRequest().GetOperationParams().labels()); - if (uid && Self->ExportsByUid.contains(uid)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Export with uid '" << uid << "' already exists" - ); - } - - const TPath domainPath = TPath::Resolve(request.GetDatabaseName(), Self); - { - TPath::TChecker checks = domainPath.Check(); - checks - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() + TEvExport::TEvCreateExportRequest::TPtr Request; + bool Progress; + + explicit TTxCreate(TSelf* self, TEvExport::TEvCreateExportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + , Progress(false) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CREATE_EXPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + LOG_D("TExport::TTxCreate: DoExecute"); + LOG_T("Message:\n" << request.ShortDebugString()); + + auto response = MakeHolder<TEvExport::TEvCreateExportResponse>(request.GetTxId()); + + const ui64 id = request.GetTxId(); + if (Self->Exports.contains(id)) { + return Reply( + std::move(response), + Ydb::StatusIds::ALREADY_EXISTS, + TStringBuilder() << "Export with id '" << id << "' already exists" + ); + } + + const TString& uid = GetUid(request.GetRequest().GetOperationParams().labels()); + if (uid && Self->ExportsByUid.contains(uid)) { + return Reply( + std::move(response), + Ydb::StatusIds::ALREADY_EXISTS, + TStringBuilder() << "Export with uid '" << uid << "' already exists" + ); + } + + const TPath domainPath = TPath::Resolve(request.GetDatabaseName(), Self); + { + TPath::TChecker checks = domainPath.Check(); + checks + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() .IsLikeDirectory(); - - if (!checks) { - TString explain; - checks.GetStatus(&explain); - - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed database check: " << explain - ); - } - } - - TExportInfo::TPtr exportInfo = nullptr; - - switch (request.GetRequest().GetSettingsCase()) { - case NKikimrExport::TCreateExportRequest::kExportToYtSettings: - { - const auto& settings = request.GetRequest().GetExportToYtSettings(); - exportInfo = new TExportInfo(id, uid, TExportInfo::EKind::YT, settings, domainPath.Base()->PathId); - - TString explain; - if (!FillItems(exportInfo, settings, explain)) { - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed item check: " << explain - ); - } - } - break; - - case NKikimrExport::TCreateExportRequest::kExportToS3Settings: - { - auto settings = request.GetRequest().GetExportToS3Settings(); - if (!settings.scheme()) { - settings.set_scheme(Ydb::Export::ExportToS3Settings::HTTPS); - } - - exportInfo = new TExportInfo(id, uid, TExportInfo::EKind::S3, settings, domainPath.Base()->PathId); - - TString explain; - if (!FillItems(exportInfo, settings, explain)) { - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed item check: " << explain - ); - } - } - break; - - default: - Y_VERIFY_DEBUG(false, "Unknown export kind"); - } - - Y_VERIFY(exportInfo != nullptr); - - if (request.HasUserSID()) { - exportInfo->UserSID = request.GetUserSID(); - } - - NIceDb::TNiceDb db(txc.DB); - Self->PersistCreateExport(db, exportInfo); - - exportInfo->State = TExportInfo::EState::CreateExportDir; - Self->PersistExportState(db, exportInfo); - - Self->Exports[id] = exportInfo; - if (uid) { - Self->ExportsByUid[uid] = exportInfo; - } - - Self->FromXxportInfo(*response->Record.MutableResponse()->MutableEntry(), exportInfo); - - Progress = true; - return Reply(std::move(response)); - } - - void DoComplete(const TActorContext& ctx) override { - LOG_D("TExport::TTxCreate: DoComplete"); - - if (Progress) { - const ui64 id = Request->Get()->Record.GetTxId(); - Self->Execute(Self->CreateTxProgressExport(id), ctx); - } - } - -private: - static TString GetUid(const google::protobuf::Map<TString, TString>& labels) { - auto it = labels.find("uid"); - if (it == labels.end()) { - return TString(); - } - - return it->second; - } - - bool Reply( - THolder<TEvExport::TEvCreateExportResponse> response, - const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS, - const TString& errorMessage = TString() - ) { - LOG_D("TExport::TTxCreate: Reply" - << ": status# " << status - << ", error# " << errorMessage); - LOG_T("Message:\n" << response->Record.ShortDebugString()); - - auto& exprt = *response->Record.MutableResponse()->MutableEntry(); - exprt.SetStatus(status); - if (errorMessage) { - AddIssue(exprt, errorMessage); - } - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - - return true; - } - - template <typename TSettings> - bool FillItems(TExportInfo::TPtr exportInfo, const TSettings& settings, TString& explain) { - exportInfo->Items.reserve(settings.items().size()); - for (ui32 itemIdx : xrange(settings.items().size())) { - const auto& item = settings.items(itemIdx); - const TPath path = TPath::Resolve(item.source_path(), Self); - { - TPath::TChecker checks = path.Check(); - checks - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsTable(); - - if (!checks) { - checks.GetStatus(&explain); - return false; - } - } - - exportInfo->Items.emplace_back(item.source_path(), path.Base()->PathId); - exportInfo->PendingItems.push_back(itemIdx); - } - - return true; - } - -}; // TTxCreate - + + if (!checks) { + TString explain; + checks.GetStatus(&explain); + + return Reply( + std::move(response), + Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Failed database check: " << explain + ); + } + } + + TExportInfo::TPtr exportInfo = nullptr; + + switch (request.GetRequest().GetSettingsCase()) { + case NKikimrExport::TCreateExportRequest::kExportToYtSettings: + { + const auto& settings = request.GetRequest().GetExportToYtSettings(); + exportInfo = new TExportInfo(id, uid, TExportInfo::EKind::YT, settings, domainPath.Base()->PathId); + + TString explain; + if (!FillItems(exportInfo, settings, explain)) { + return Reply( + std::move(response), + Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Failed item check: " << explain + ); + } + } + break; + + case NKikimrExport::TCreateExportRequest::kExportToS3Settings: + { + auto settings = request.GetRequest().GetExportToS3Settings(); + if (!settings.scheme()) { + settings.set_scheme(Ydb::Export::ExportToS3Settings::HTTPS); + } + + exportInfo = new TExportInfo(id, uid, TExportInfo::EKind::S3, settings, domainPath.Base()->PathId); + + TString explain; + if (!FillItems(exportInfo, settings, explain)) { + return Reply( + std::move(response), + Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Failed item check: " << explain + ); + } + } + break; + + default: + Y_VERIFY_DEBUG(false, "Unknown export kind"); + } + + Y_VERIFY(exportInfo != nullptr); + + if (request.HasUserSID()) { + exportInfo->UserSID = request.GetUserSID(); + } + + NIceDb::TNiceDb db(txc.DB); + Self->PersistCreateExport(db, exportInfo); + + exportInfo->State = TExportInfo::EState::CreateExportDir; + Self->PersistExportState(db, exportInfo); + + Self->Exports[id] = exportInfo; + if (uid) { + Self->ExportsByUid[uid] = exportInfo; + } + + Self->FromXxportInfo(*response->Record.MutableResponse()->MutableEntry(), exportInfo); + + Progress = true; + return Reply(std::move(response)); + } + + void DoComplete(const TActorContext& ctx) override { + LOG_D("TExport::TTxCreate: DoComplete"); + + if (Progress) { + const ui64 id = Request->Get()->Record.GetTxId(); + Self->Execute(Self->CreateTxProgressExport(id), ctx); + } + } + +private: + static TString GetUid(const google::protobuf::Map<TString, TString>& labels) { + auto it = labels.find("uid"); + if (it == labels.end()) { + return TString(); + } + + return it->second; + } + + bool Reply( + THolder<TEvExport::TEvCreateExportResponse> response, + const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS, + const TString& errorMessage = TString() + ) { + LOG_D("TExport::TTxCreate: Reply" + << ": status# " << status + << ", error# " << errorMessage); + LOG_T("Message:\n" << response->Record.ShortDebugString()); + + auto& exprt = *response->Record.MutableResponse()->MutableEntry(); + exprt.SetStatus(status); + if (errorMessage) { + AddIssue(exprt, errorMessage); + } + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + + return true; + } + + template <typename TSettings> + bool FillItems(TExportInfo::TPtr exportInfo, const TSettings& settings, TString& explain) { + exportInfo->Items.reserve(settings.items().size()); + for (ui32 itemIdx : xrange(settings.items().size())) { + const auto& item = settings.items(itemIdx); + const TPath path = TPath::Resolve(item.source_path(), Self); + { + TPath::TChecker checks = path.Check(); + checks + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsTable(); + + if (!checks) { + checks.GetStatus(&explain); + return false; + } + } + + exportInfo->Items.emplace_back(item.source_path(), path.Base()->PathId); + exportInfo->PendingItems.push_back(itemIdx); + } + + return true; + } + +}; // TTxCreate + struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase { - using EState = TExportInfo::EState; - using ESubState = TExportInfo::TItem::ESubState; - - static constexpr ui32 IssuesSizeLimit = 2 * 1024; - - ui64 Id; - TEvTxAllocatorClient::TEvAllocateResult::TPtr AllocateResult = nullptr; + using EState = TExportInfo::EState; + using ESubState = TExportInfo::TItem::ESubState; + + static constexpr ui32 IssuesSizeLimit = 2 * 1024; + + ui64 Id; + TEvTxAllocatorClient::TEvAllocateResult::TPtr AllocateResult = nullptr; TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr ModifyResult = nullptr; TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr NotifyResult = nullptr; - - explicit TTxProgress(TSelf* self, ui64 id) - : TXxport::TTxBase(self) - , Id(id) - { - } - - explicit TTxProgress(TSelf* self, TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) - : TXxport::TTxBase(self) - , AllocateResult(ev) - { - } - + + explicit TTxProgress(TSelf* self, ui64 id) + : TXxport::TTxBase(self) + , Id(id) + { + } + + explicit TTxProgress(TSelf* self, TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) + : TXxport::TTxBase(self) + , AllocateResult(ev) + { + } + explicit TTxProgress(TSelf* self, TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) - : TXxport::TTxBase(self) - , ModifyResult(ev) - { - } - + : TXxport::TTxBase(self) + , ModifyResult(ev) + { + } + explicit TTxProgress(TSelf* self, TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) - : TXxport::TTxBase(self) - , NotifyResult(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_EXPORT_PROGRESS; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_D("TExport::TTxProgress: DoExecute"); - - if (AllocateResult) { - OnAllocateResult(txc, ctx); - } else if (ModifyResult) { - OnModifyResult(txc, ctx); - } else if (NotifyResult) { - OnNotifyResult(txc, ctx); - } else { - Resume(txc, ctx); - } - - return true; - } - - void DoComplete(const TActorContext&) override { - LOG_D("TExport::TTxProgress: DoComplete"); - } - -private: - void MkDir(TExportInfo::TPtr exportInfo, TTxId txId) { - LOG_I("TExport::TTxProgress: MkDir propose" - << ": info# " << exportInfo->ToString() - << ", txId# " << txId); - - Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); - Send(Self->SelfId(), MkDirPropose(Self, txId, exportInfo)); - } - - void CopyTables(TExportInfo::TPtr exportInfo, TTxId txId) { - LOG_I("TExport::TTxProgress: CopyTables propose" - << ": info# " << exportInfo->ToString() - << ", txId# " << txId); - - Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); - Send(Self->SelfId(), CopyTablesPropose(Self, txId, exportInfo)); - } - - void TransferData(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - auto& item = exportInfo->Items.at(itemIdx); - - item.SubState = ESubState::Proposed; - - LOG_I("TExport::TTxProgress: Backup propose" - << ": info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->SelfId(), BackupPropose(Self, txId, exportInfo, itemIdx)); - } - - bool CancelTransferring(TExportInfo::TPtr exportInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - const auto& item = exportInfo->Items.at(itemIdx); - - if (item.WaitTxId == InvalidTxId) { - if (item.SubState == ESubState::Proposed) { - exportInfo->State = EState::Cancellation; - } - - return false; - } - - exportInfo->State = EState::Cancellation; - - LOG_I("TExport::TTxProgress: cancel backup's tx" - << ": info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Send(Self->SelfId(), CancelPropose(exportInfo, item.WaitTxId), 0, exportInfo->Id); - return true; - } - - void DropTable(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - const auto& item = exportInfo->Items.at(itemIdx); - - LOG_I("TExport::TTxProgress: Drop propose" - << ": info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->SelfId(), DropPropose(Self, txId, exportInfo, itemIdx)); - } - - void DropDir(TExportInfo::TPtr exportInfo, TTxId txId) { - LOG_I("TExport::TTxProgress: Drop propose" - << ": info# " << exportInfo->ToString() - << ", txId# " << txId); - - Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); - Send(Self->SelfId(), DropPropose(Self, txId, exportInfo)); - } - - void AllocateTxId(TExportInfo::TPtr exportInfo) { - LOG_I("TExport::TTxProgress: Allocate txId" - << ": info# " << exportInfo->ToString()); - - Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); - Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, exportInfo->Id); - } - - void AllocateTxId(TExportInfo::TPtr exportInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - auto& item = exportInfo->Items.at(itemIdx); - - item.SubState = ESubState::AllocateTxId; - - LOG_I("TExport::TTxProgress: Allocate txId" - << ": info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, exportInfo->Id); - } - - void SubscribeTx(TTxId txId) { + : TXxport::TTxBase(self) + , NotifyResult(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_EXPORT_PROGRESS; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_D("TExport::TTxProgress: DoExecute"); + + if (AllocateResult) { + OnAllocateResult(txc, ctx); + } else if (ModifyResult) { + OnModifyResult(txc, ctx); + } else if (NotifyResult) { + OnNotifyResult(txc, ctx); + } else { + Resume(txc, ctx); + } + + return true; + } + + void DoComplete(const TActorContext&) override { + LOG_D("TExport::TTxProgress: DoComplete"); + } + +private: + void MkDir(TExportInfo::TPtr exportInfo, TTxId txId) { + LOG_I("TExport::TTxProgress: MkDir propose" + << ": info# " << exportInfo->ToString() + << ", txId# " << txId); + + Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); + Send(Self->SelfId(), MkDirPropose(Self, txId, exportInfo)); + } + + void CopyTables(TExportInfo::TPtr exportInfo, TTxId txId) { + LOG_I("TExport::TTxProgress: CopyTables propose" + << ": info# " << exportInfo->ToString() + << ", txId# " << txId); + + Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); + Send(Self->SelfId(), CopyTablesPropose(Self, txId, exportInfo)); + } + + void TransferData(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto& item = exportInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TExport::TTxProgress: Backup propose" + << ": info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->SelfId(), BackupPropose(Self, txId, exportInfo, itemIdx)); + } + + bool CancelTransferring(TExportInfo::TPtr exportInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + const auto& item = exportInfo->Items.at(itemIdx); + + if (item.WaitTxId == InvalidTxId) { + if (item.SubState == ESubState::Proposed) { + exportInfo->State = EState::Cancellation; + } + + return false; + } + + exportInfo->State = EState::Cancellation; + + LOG_I("TExport::TTxProgress: cancel backup's tx" + << ": info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Send(Self->SelfId(), CancelPropose(exportInfo, item.WaitTxId), 0, exportInfo->Id); + return true; + } + + void DropTable(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + const auto& item = exportInfo->Items.at(itemIdx); + + LOG_I("TExport::TTxProgress: Drop propose" + << ": info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->SelfId(), DropPropose(Self, txId, exportInfo, itemIdx)); + } + + void DropDir(TExportInfo::TPtr exportInfo, TTxId txId) { + LOG_I("TExport::TTxProgress: Drop propose" + << ": info# " << exportInfo->ToString() + << ", txId# " << txId); + + Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); + Send(Self->SelfId(), DropPropose(Self, txId, exportInfo)); + } + + void AllocateTxId(TExportInfo::TPtr exportInfo) { + LOG_I("TExport::TTxProgress: Allocate txId" + << ": info# " << exportInfo->ToString()); + + Y_VERIFY(exportInfo->WaitTxId == InvalidTxId); + Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, exportInfo->Id); + } + + void AllocateTxId(TExportInfo::TPtr exportInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto& item = exportInfo->Items.at(itemIdx); + + item.SubState = ESubState::AllocateTxId; + + LOG_I("TExport::TTxProgress: Allocate txId" + << ": info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, exportInfo->Id); + } + + void SubscribeTx(TTxId txId) { Send(Self->SelfId(), new TEvSchemeShard::TEvNotifyTxCompletion(ui64(txId))); - } - - void SubscribeTx(TExportInfo::TPtr exportInfo) { - LOG_I("TExport::TTxProgress: Wait for completion" - << ": info# " << exportInfo->ToString()); - - Y_VERIFY(exportInfo->WaitTxId != InvalidTxId); - SubscribeTx(exportInfo->WaitTxId); - } - - void SubscribeTx(TExportInfo::TPtr exportInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - auto& item = exportInfo->Items.at(itemIdx); - - item.SubState = ESubState::Subscribed; - - LOG_I("TExport::TTxProgress: Wait for completion" - << ": info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Y_VERIFY(item.WaitTxId != InvalidTxId); - SubscribeTx(item.WaitTxId); - } - + } + + void SubscribeTx(TExportInfo::TPtr exportInfo) { + LOG_I("TExport::TTxProgress: Wait for completion" + << ": info# " << exportInfo->ToString()); + + Y_VERIFY(exportInfo->WaitTxId != InvalidTxId); + SubscribeTx(exportInfo->WaitTxId); + } + + void SubscribeTx(TExportInfo::TPtr exportInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto& item = exportInfo->Items.at(itemIdx); + + item.SubState = ESubState::Subscribed; + + LOG_I("TExport::TTxProgress: Wait for completion" + << ": info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Y_VERIFY(item.WaitTxId != InvalidTxId); + SubscribeTx(item.WaitTxId); + } + static TPathId ItemPathId(TSchemeShard* ss, TExportInfo::TPtr exportInfo, ui32 itemIdx) { - const TPath itemPath = TPath::Resolve(ExportItemPathName(ss, exportInfo, itemIdx), ss); - - if (!itemPath.IsResolved()) { - return {}; - } - - return itemPath.Base()->PathId; - } - - TTxId GetActiveCopyingTxId(TExportInfo::TPtr exportInfo) { - if (exportInfo->Items.size() < 1) { - return InvalidTxId; - } - - const auto& item = exportInfo->Items.at(0); - if (!Self->PathsById.contains(item.SourcePathId)) { - return InvalidTxId; - } - - auto path = Self->PathsById.at(item.SourcePathId); + const TPath itemPath = TPath::Resolve(ExportItemPathName(ss, exportInfo, itemIdx), ss); + + if (!itemPath.IsResolved()) { + return {}; + } + + return itemPath.Base()->PathId; + } + + TTxId GetActiveCopyingTxId(TExportInfo::TPtr exportInfo) { + if (exportInfo->Items.size() < 1) { + return InvalidTxId; + } + + const auto& item = exportInfo->Items.at(0); + if (!Self->PathsById.contains(item.SourcePathId)) { + return InvalidTxId; + } + + auto path = Self->PathsById.at(item.SourcePathId); if (path->PathState != NKikimrSchemeOp::EPathStateCopying) { - return InvalidTxId; - } - - return path->LastTxId; - } - - TTxId GetActiveBackupTxId(TExportInfo::TPtr exportInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - const auto& item = exportInfo->Items.at(itemIdx); - - Y_VERIFY(item.State == EState::Transferring); - - const auto itemPathId = ItemPathId(Self, exportInfo, itemIdx); - if (!itemPathId) { - return InvalidTxId; - } - - if (!Self->PathsById.contains(itemPathId)) { - return InvalidTxId; - } - - auto path = Self->PathsById.at(itemPathId); + return InvalidTxId; + } + + return path->LastTxId; + } + + TTxId GetActiveBackupTxId(TExportInfo::TPtr exportInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + const auto& item = exportInfo->Items.at(itemIdx); + + Y_VERIFY(item.State == EState::Transferring); + + const auto itemPathId = ItemPathId(Self, exportInfo, itemIdx); + if (!itemPathId) { + return InvalidTxId; + } + + if (!Self->PathsById.contains(itemPathId)) { + return InvalidTxId; + } + + auto path = Self->PathsById.at(itemPathId); if (path->PathState != NKikimrSchemeOp::EPathStateBackup) { - return InvalidTxId; - } - - return path->LastTxId; - } - - void Cancel(TExportInfo::TPtr exportInfo, ui32 itemIdx, TStringBuf marker) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - const auto& item = exportInfo->Items.at(itemIdx); - - LOG_N("TExport::TTxProgress: " << marker << ", cancelling" - << ", info# " << exportInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - exportInfo->State = EState::Cancelled; - - for (ui32 i : xrange(exportInfo->Items.size())) { - if (i == itemIdx) { - continue; - } - - if (exportInfo->Items.at(i).State != EState::Transferring) { - continue; - } - - CancelTransferring(exportInfo, i); - } - } - - TMaybe<TString> GetIssues(const TPathId& itemPathId, TTxId backupTxId) { - if (!Self->Tables.contains(itemPathId)) { - return TStringBuilder() << "Cannot find table: " << itemPathId; - } - - TTableInfo::TPtr table = Self->Tables.at(itemPathId); - if (!table->BackupHistory.contains(backupTxId)) { - return TStringBuilder() << "Cannot find backup: " << backupTxId << " for table: " << itemPathId; - } - - const auto& result = table->BackupHistory.at(backupTxId); - if (result.TotalShardCount == result.SuccessShardCount) { - return Nothing(); - } - - TStringBuilder output; - bool first = true; - - for (const auto& [shardId, status] : result.ShardStatuses) { - if (status.Success) { - continue; - } - - if (output.size() > IssuesSizeLimit) { - output << "... <truncated>"; - break; - } - - if (!first) { - output << ", "; - } - first = false; - - output << "shard: " << shardId << ", error: " << status.Error; - } - - return output; - } - - void Resume(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(Self->Exports.contains(Id)); - TExportInfo::TPtr exportInfo = Self->Exports.at(Id); - - LOG_D("TExport::TTxProgress: Resume" - << ": id# " << Id); - - NIceDb::TNiceDb db(txc.DB); - - switch (exportInfo->State) { - case EState::CreateExportDir: - case EState::CopyTables: - if (exportInfo->WaitTxId == InvalidTxId) { - AllocateTxId(exportInfo); - } else { - SubscribeTx(exportInfo); - } - break; - - case EState::Transferring: - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - const auto& item = exportInfo->Items.at(itemIdx); - - if (item.WaitTxId == InvalidTxId) { - AllocateTxId(exportInfo, itemIdx); - } else { - SubscribeTx(exportInfo, itemIdx); - } - } - break; - - case EState::Cancellation: - exportInfo->State = EState::Cancelled; - // will be switched back to Cancellation if there is any active backup tx - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - auto& item = exportInfo->Items.at(itemIdx); - - if (item.State != EState::Transferring) { - continue; - } - - if (!CancelTransferring(exportInfo, itemIdx)) { - const TTxId txId = GetActiveBackupTxId(exportInfo, itemIdx); - - if (txId == InvalidTxId) { - item.State = EState::Cancelled; - } else { - item.WaitTxId = txId; - CancelTransferring(exportInfo, itemIdx); - } - - Self->PersistExportItemState(db, exportInfo, itemIdx); - } - } - - Self->PersistExportState(db, exportInfo); - SendNotificationsIfFinished(exportInfo); - break; - - case EState::Dropping: - if (!exportInfo->AllItemsAreDropped()) { - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - const auto& item = exportInfo->Items.at(itemIdx); - - if (item.State != EState::Dropping) { - continue; - } - - if (item.WaitTxId == InvalidTxId) { - exportInfo->PendingDropItems.push_back(itemIdx); - AllocateTxId(exportInfo, itemIdx); - } else { - SubscribeTx(exportInfo, itemIdx); - } - } - } else { - if (exportInfo->WaitTxId == InvalidTxId) { - AllocateTxId(exportInfo); - } else { - SubscribeTx(exportInfo); - } - } - break; - - default: - break; - } - } - - void OnAllocateResult(TTransactionContext&, const TActorContext&) { - Y_VERIFY(AllocateResult); - + return InvalidTxId; + } + + return path->LastTxId; + } + + void Cancel(TExportInfo::TPtr exportInfo, ui32 itemIdx, TStringBuf marker) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + const auto& item = exportInfo->Items.at(itemIdx); + + LOG_N("TExport::TTxProgress: " << marker << ", cancelling" + << ", info# " << exportInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + exportInfo->State = EState::Cancelled; + + for (ui32 i : xrange(exportInfo->Items.size())) { + if (i == itemIdx) { + continue; + } + + if (exportInfo->Items.at(i).State != EState::Transferring) { + continue; + } + + CancelTransferring(exportInfo, i); + } + } + + TMaybe<TString> GetIssues(const TPathId& itemPathId, TTxId backupTxId) { + if (!Self->Tables.contains(itemPathId)) { + return TStringBuilder() << "Cannot find table: " << itemPathId; + } + + TTableInfo::TPtr table = Self->Tables.at(itemPathId); + if (!table->BackupHistory.contains(backupTxId)) { + return TStringBuilder() << "Cannot find backup: " << backupTxId << " for table: " << itemPathId; + } + + const auto& result = table->BackupHistory.at(backupTxId); + if (result.TotalShardCount == result.SuccessShardCount) { + return Nothing(); + } + + TStringBuilder output; + bool first = true; + + for (const auto& [shardId, status] : result.ShardStatuses) { + if (status.Success) { + continue; + } + + if (output.size() > IssuesSizeLimit) { + output << "... <truncated>"; + break; + } + + if (!first) { + output << ", "; + } + first = false; + + output << "shard: " << shardId << ", error: " << status.Error; + } + + return output; + } + + void Resume(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(Self->Exports.contains(Id)); + TExportInfo::TPtr exportInfo = Self->Exports.at(Id); + + LOG_D("TExport::TTxProgress: Resume" + << ": id# " << Id); + + NIceDb::TNiceDb db(txc.DB); + + switch (exportInfo->State) { + case EState::CreateExportDir: + case EState::CopyTables: + if (exportInfo->WaitTxId == InvalidTxId) { + AllocateTxId(exportInfo); + } else { + SubscribeTx(exportInfo); + } + break; + + case EState::Transferring: + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + const auto& item = exportInfo->Items.at(itemIdx); + + if (item.WaitTxId == InvalidTxId) { + AllocateTxId(exportInfo, itemIdx); + } else { + SubscribeTx(exportInfo, itemIdx); + } + } + break; + + case EState::Cancellation: + exportInfo->State = EState::Cancelled; + // will be switched back to Cancellation if there is any active backup tx + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + auto& item = exportInfo->Items.at(itemIdx); + + if (item.State != EState::Transferring) { + continue; + } + + if (!CancelTransferring(exportInfo, itemIdx)) { + const TTxId txId = GetActiveBackupTxId(exportInfo, itemIdx); + + if (txId == InvalidTxId) { + item.State = EState::Cancelled; + } else { + item.WaitTxId = txId; + CancelTransferring(exportInfo, itemIdx); + } + + Self->PersistExportItemState(db, exportInfo, itemIdx); + } + } + + Self->PersistExportState(db, exportInfo); + SendNotificationsIfFinished(exportInfo); + break; + + case EState::Dropping: + if (!exportInfo->AllItemsAreDropped()) { + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + const auto& item = exportInfo->Items.at(itemIdx); + + if (item.State != EState::Dropping) { + continue; + } + + if (item.WaitTxId == InvalidTxId) { + exportInfo->PendingDropItems.push_back(itemIdx); + AllocateTxId(exportInfo, itemIdx); + } else { + SubscribeTx(exportInfo, itemIdx); + } + } + } else { + if (exportInfo->WaitTxId == InvalidTxId) { + AllocateTxId(exportInfo); + } else { + SubscribeTx(exportInfo); + } + } + break; + + default: + break; + } + } + + void OnAllocateResult(TTransactionContext&, const TActorContext&) { + Y_VERIFY(AllocateResult); + const auto txId = TTxId(AllocateResult->Get()->TxIds.front()); - const ui64 id = AllocateResult->Cookie; - - LOG_D("TExport::TTxProgress: OnAllocateResult" - << ": txId# " << txId - << ", id# " << id); - - if (!Self->Exports.contains(id)) { - LOG_E("TExport::TTxProgress: OnAllocateResult received unknown id" - << ": id# " << id); - return; - } - - TExportInfo::TPtr exportInfo = Self->Exports.at(id); - ui32 itemIdx = Max<ui32>(); - - auto popPendingItemIdx = [](TDeque<ui32>& pendingItems) { - const ui32 itemIdx = pendingItems.front(); - pendingItems.pop_front(); - return itemIdx; - }; - - switch (exportInfo->State) { - case EState::CreateExportDir: - MkDir(exportInfo, txId); - break; - - case EState::CopyTables: - CopyTables(exportInfo, txId); - break; - - case EState::Transferring: - if (exportInfo->PendingItems) { - itemIdx = popPendingItemIdx(exportInfo->PendingItems); - TransferData(exportInfo, itemIdx, txId); - } else { - return; - } - break; - - case EState::Dropping: - if (exportInfo->PendingDropItems) { - itemIdx = popPendingItemIdx(exportInfo->PendingDropItems); - DropTable(exportInfo, itemIdx, txId); - } else { - DropDir(exportInfo, txId); - } - break; - - default: - return; - } - - Y_VERIFY(!Self->TxIdToExport.contains(txId)); - Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; - } - - void OnModifyResult(TTransactionContext& txc, const TActorContext& ctx) { - Y_VERIFY(ModifyResult); - const auto& record = ModifyResult->Get()->Record; - - LOG_D("TExport::TTxProgress: OnModifyResult" - << ": txId# " << record.GetTxId() - << ", status# " << record.GetStatus()); - LOG_T("Message:\n" << record.ShortDebugString()); - - auto txId = TTxId(record.GetTxId()); - if (!Self->TxIdToExport.contains(txId)) { - LOG_E("TExport::TTxProgress: OnModifyResult received unknown txId" - << ": txId# " << txId); - return; - } - - ui64 id; - ui32 itemIdx; - std::tie(id, itemIdx) = Self->TxIdToExport.at(txId); - if (!Self->Exports.contains(id)) { - LOG_E("TExport::TTxProgress: OnModifyResult received unknown id" - << ": id# " << id); - return; - } - - TExportInfo::TPtr exportInfo = Self->Exports.at(id); - NIceDb::TNiceDb db(txc.DB); - + const ui64 id = AllocateResult->Cookie; + + LOG_D("TExport::TTxProgress: OnAllocateResult" + << ": txId# " << txId + << ", id# " << id); + + if (!Self->Exports.contains(id)) { + LOG_E("TExport::TTxProgress: OnAllocateResult received unknown id" + << ": id# " << id); + return; + } + + TExportInfo::TPtr exportInfo = Self->Exports.at(id); + ui32 itemIdx = Max<ui32>(); + + auto popPendingItemIdx = [](TDeque<ui32>& pendingItems) { + const ui32 itemIdx = pendingItems.front(); + pendingItems.pop_front(); + return itemIdx; + }; + + switch (exportInfo->State) { + case EState::CreateExportDir: + MkDir(exportInfo, txId); + break; + + case EState::CopyTables: + CopyTables(exportInfo, txId); + break; + + case EState::Transferring: + if (exportInfo->PendingItems) { + itemIdx = popPendingItemIdx(exportInfo->PendingItems); + TransferData(exportInfo, itemIdx, txId); + } else { + return; + } + break; + + case EState::Dropping: + if (exportInfo->PendingDropItems) { + itemIdx = popPendingItemIdx(exportInfo->PendingDropItems); + DropTable(exportInfo, itemIdx, txId); + } else { + DropDir(exportInfo, txId); + } + break; + + default: + return; + } + + Y_VERIFY(!Self->TxIdToExport.contains(txId)); + Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; + } + + void OnModifyResult(TTransactionContext& txc, const TActorContext& ctx) { + Y_VERIFY(ModifyResult); + const auto& record = ModifyResult->Get()->Record; + + LOG_D("TExport::TTxProgress: OnModifyResult" + << ": txId# " << record.GetTxId() + << ", status# " << record.GetStatus()); + LOG_T("Message:\n" << record.ShortDebugString()); + + auto txId = TTxId(record.GetTxId()); + if (!Self->TxIdToExport.contains(txId)) { + LOG_E("TExport::TTxProgress: OnModifyResult received unknown txId" + << ": txId# " << txId); + return; + } + + ui64 id; + ui32 itemIdx; + std::tie(id, itemIdx) = Self->TxIdToExport.at(txId); + if (!Self->Exports.contains(id)) { + LOG_E("TExport::TTxProgress: OnModifyResult received unknown id" + << ": id# " << id); + return; + } + + TExportInfo::TPtr exportInfo = Self->Exports.at(id); + NIceDb::TNiceDb db(txc.DB); + if (record.GetStatus() != NKikimrScheme::StatusAccepted) { - Self->TxIdToExport.erase(txId); - txId = InvalidTxId; - - const auto status = record.GetStatus(); + Self->TxIdToExport.erase(txId); + txId = InvalidTxId; + + const auto status = record.GetStatus(); const bool isMultipleMods = status == NKikimrScheme::StatusMultipleModifications; const bool isAlreadyExists = status == NKikimrScheme::StatusAlreadyExists; const bool isNotExist = status == NKikimrScheme::StatusPathDoesNotExist; - - switch (exportInfo->State) { - case EState::CreateExportDir: - case EState::CopyTables: - if (isMultipleMods || isAlreadyExists) { - if (record.GetPathCreateTxId()) { - txId = TTxId(record.GetPathCreateTxId()); - } else { - txId = GetActiveCopyingTxId(exportInfo); - } - } - break; - - case EState::Transferring: - if (isMultipleMods) { - txId = GetActiveBackupTxId(exportInfo, itemIdx); - } - break; - - case EState::Dropping: - if (isMultipleMods || isNotExist) { - if (record.GetPathDropTxId()) { + + switch (exportInfo->State) { + case EState::CreateExportDir: + case EState::CopyTables: + if (isMultipleMods || isAlreadyExists) { + if (record.GetPathCreateTxId()) { + txId = TTxId(record.GetPathCreateTxId()); + } else { + txId = GetActiveCopyingTxId(exportInfo); + } + } + break; + + case EState::Transferring: + if (isMultipleMods) { + txId = GetActiveBackupTxId(exportInfo, itemIdx); + } + break; + + case EState::Dropping: + if (isMultipleMods || isNotExist) { + if (record.GetPathDropTxId()) { // We may need to wait for an earlier tx - txId = TTxId(record.GetPathDropTxId()); + txId = TTxId(record.GetPathDropTxId()); } else if (isNotExist) { // Already dropped and fully forgotten txId = TTxId(record.GetTxId()); - } else { + } else { // We need to wait and retry the operation - txId = TTxId(record.GetTxId()); - THolder<IEventBase> ev; - - if (itemIdx < exportInfo->Items.size()) { - ev.Reset(DropPropose(Self, txId, exportInfo, itemIdx).Release()); - } else { - ev.Reset(DropPropose(Self, txId, exportInfo).Release()); - } - - ctx.TActivationContext::Schedule(TDuration::Seconds(10), - new IEventHandle(Self->SelfId(), Self->SelfId(), ev.Release())); - Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; - - return; - } - } - break; - - default: - break; - } - - if (txId == InvalidTxId) { - if (itemIdx < exportInfo->Items.size()) { - auto& item = exportInfo->Items.at(itemIdx); - - item.State = EState::Cancelled; - item.Issue = record.GetReason(); - Self->PersistExportItemState(db, exportInfo, itemIdx); - - if (!exportInfo->IsInProgress()) { - return; - } - - Cancel(exportInfo, itemIdx, "unhappy propose"); - } else { - if (!exportInfo->IsInProgress()) { - return; - } - - exportInfo->Issue = record.GetReason(); - exportInfo->State = EState::Cancelled; - } - - Self->PersistExportState(db, exportInfo); - return SendNotificationsIfFinished(exportInfo); - } - - Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; - } - - switch (exportInfo->State) { - case EState::CreateExportDir: + txId = TTxId(record.GetTxId()); + THolder<IEventBase> ev; + + if (itemIdx < exportInfo->Items.size()) { + ev.Reset(DropPropose(Self, txId, exportInfo, itemIdx).Release()); + } else { + ev.Reset(DropPropose(Self, txId, exportInfo).Release()); + } + + ctx.TActivationContext::Schedule(TDuration::Seconds(10), + new IEventHandle(Self->SelfId(), Self->SelfId(), ev.Release())); + Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; + + return; + } + } + break; + + default: + break; + } + + if (txId == InvalidTxId) { + if (itemIdx < exportInfo->Items.size()) { + auto& item = exportInfo->Items.at(itemIdx); + + item.State = EState::Cancelled; + item.Issue = record.GetReason(); + Self->PersistExportItemState(db, exportInfo, itemIdx); + + if (!exportInfo->IsInProgress()) { + return; + } + + Cancel(exportInfo, itemIdx, "unhappy propose"); + } else { + if (!exportInfo->IsInProgress()) { + return; + } + + exportInfo->Issue = record.GetReason(); + exportInfo->State = EState::Cancelled; + } + + Self->PersistExportState(db, exportInfo); + return SendNotificationsIfFinished(exportInfo); + } + + Self->TxIdToExport[txId] = {exportInfo->Id, itemIdx}; + } + + switch (exportInfo->State) { + case EState::CreateExportDir: exportInfo->ExportPathId = Self->MakeLocalId(TLocalPathId(record.GetPathId())); - Self->PersistExportPathId(db, exportInfo); - - exportInfo->WaitTxId = txId; - Self->PersistExportState(db, exportInfo); - break; - - case EState::CopyTables: - exportInfo->WaitTxId = txId; - Self->PersistExportState(db, exportInfo); - break; - - case EState::Transferring: - Y_VERIFY(itemIdx < exportInfo->Items.size()); - exportInfo->Items.at(itemIdx).WaitTxId = txId; - Self->PersistExportItemState(db, exportInfo, itemIdx); - break; - - case EState::Dropping: - if (!exportInfo->AllItemsAreDropped()) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - exportInfo->Items.at(itemIdx).WaitTxId = txId; - Self->PersistExportItemState(db, exportInfo, itemIdx); - } else { - exportInfo->WaitTxId = txId; - Self->PersistExportState(db, exportInfo); - } - break; - - case EState::Cancellation: - if (itemIdx < exportInfo->Items.size()) { - exportInfo->Items.at(itemIdx).WaitTxId = txId; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - CancelTransferring(exportInfo, itemIdx); - } - return; - - default: - return; // no need to wait notification - } - - LOG_I("TExport::TTxProgress: Wait for completion" - << ": info# " << exportInfo->ToString() - << ", itemIdx# " << itemIdx - << ", txId# " << txId); - SubscribeTx(txId); - } - - void OnNotifyResult(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(NotifyResult); - const auto& record = NotifyResult->Get()->Record; - - LOG_D("TExport::TTxProgress: OnNotifyResult" - << ": txId# " << record.GetTxId()); - + Self->PersistExportPathId(db, exportInfo); + + exportInfo->WaitTxId = txId; + Self->PersistExportState(db, exportInfo); + break; + + case EState::CopyTables: + exportInfo->WaitTxId = txId; + Self->PersistExportState(db, exportInfo); + break; + + case EState::Transferring: + Y_VERIFY(itemIdx < exportInfo->Items.size()); + exportInfo->Items.at(itemIdx).WaitTxId = txId; + Self->PersistExportItemState(db, exportInfo, itemIdx); + break; + + case EState::Dropping: + if (!exportInfo->AllItemsAreDropped()) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + exportInfo->Items.at(itemIdx).WaitTxId = txId; + Self->PersistExportItemState(db, exportInfo, itemIdx); + } else { + exportInfo->WaitTxId = txId; + Self->PersistExportState(db, exportInfo); + } + break; + + case EState::Cancellation: + if (itemIdx < exportInfo->Items.size()) { + exportInfo->Items.at(itemIdx).WaitTxId = txId; + Self->PersistExportItemState(db, exportInfo, itemIdx); + + CancelTransferring(exportInfo, itemIdx); + } + return; + + default: + return; // no need to wait notification + } + + LOG_I("TExport::TTxProgress: Wait for completion" + << ": info# " << exportInfo->ToString() + << ", itemIdx# " << itemIdx + << ", txId# " << txId); + SubscribeTx(txId); + } + + void OnNotifyResult(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(NotifyResult); + const auto& record = NotifyResult->Get()->Record; + + LOG_D("TExport::TTxProgress: OnNotifyResult" + << ": txId# " << record.GetTxId()); + const auto txId = TTxId(record.GetTxId()); - if (!Self->TxIdToExport.contains(txId)) { - LOG_E("TExport::TTxProgress: OnNotifyResult received unknown txId" - << ": txId# " << txId); - return; - } - - ui64 id; - ui32 itemIdx; - std::tie(id, itemIdx) = Self->TxIdToExport.at(txId); - if (!Self->Exports.contains(id)) { - LOG_E("TExport::TTxProgress: OnNotifyResult received unknown id" - << ": id# " << id); - return; - } - - Self->TxIdToExport.erase(txId); - - TExportInfo::TPtr exportInfo = Self->Exports.at(id); - NIceDb::TNiceDb db(txc.DB); - - switch (exportInfo->State) { - case EState::CreateExportDir: - exportInfo->State = EState::CopyTables; - exportInfo->WaitTxId = InvalidTxId; - AllocateTxId(exportInfo); - break; - - case EState::CopyTables: - exportInfo->State = EState::Transferring; - exportInfo->WaitTxId = InvalidTxId; - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - exportInfo->Items.at(itemIdx).State = EState::Transferring; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - AllocateTxId(exportInfo, itemIdx); - } - break; - - case EState::Transferring: { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - auto& item = exportInfo->Items.at(itemIdx); - - item.State = EState::Done; - item.WaitTxId = InvalidTxId; - - if (const auto issue = GetIssues(ItemPathId(Self, exportInfo, itemIdx), txId)) { - item.Issue = *issue; - Cancel(exportInfo, itemIdx, "issues during backing up"); - } else { - if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) { - exportInfo->State = EState::Done; - } - } - - Self->PersistExportItemState(db, exportInfo, itemIdx); - break; - } - - case EState::Dropping: - if (!exportInfo->AllItemsAreDropped()) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - auto& item = exportInfo->Items.at(itemIdx); - - item.State = EState::Dropped; - item.WaitTxId = InvalidTxId; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - if (exportInfo->AllItemsAreDropped()) { - AllocateTxId(exportInfo); - } - } else { - SendNotificationsIfFinished(exportInfo, true); // for tests - - if (exportInfo->Uid) { - Self->ExportsByUid.erase(exportInfo->Uid); - } - - Self->Exports.erase(exportInfo->Id); - Self->PersistRemoveExport(db, exportInfo); - } - return; - - default: + if (!Self->TxIdToExport.contains(txId)) { + LOG_E("TExport::TTxProgress: OnNotifyResult received unknown txId" + << ": txId# " << txId); + return; + } + + ui64 id; + ui32 itemIdx; + std::tie(id, itemIdx) = Self->TxIdToExport.at(txId); + if (!Self->Exports.contains(id)) { + LOG_E("TExport::TTxProgress: OnNotifyResult received unknown id" + << ": id# " << id); + return; + } + + Self->TxIdToExport.erase(txId); + + TExportInfo::TPtr exportInfo = Self->Exports.at(id); + NIceDb::TNiceDb db(txc.DB); + + switch (exportInfo->State) { + case EState::CreateExportDir: + exportInfo->State = EState::CopyTables; + exportInfo->WaitTxId = InvalidTxId; + AllocateTxId(exportInfo); + break; + + case EState::CopyTables: + exportInfo->State = EState::Transferring; + exportInfo->WaitTxId = InvalidTxId; + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + exportInfo->Items.at(itemIdx).State = EState::Transferring; + Self->PersistExportItemState(db, exportInfo, itemIdx); + + AllocateTxId(exportInfo, itemIdx); + } + break; + + case EState::Transferring: { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto& item = exportInfo->Items.at(itemIdx); + + item.State = EState::Done; + item.WaitTxId = InvalidTxId; + + if (const auto issue = GetIssues(ItemPathId(Self, exportInfo, itemIdx), txId)) { + item.Issue = *issue; + Cancel(exportInfo, itemIdx, "issues during backing up"); + } else { + if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) { + exportInfo->State = EState::Done; + } + } + + Self->PersistExportItemState(db, exportInfo, itemIdx); + break; + } + + case EState::Dropping: + if (!exportInfo->AllItemsAreDropped()) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto& item = exportInfo->Items.at(itemIdx); + + item.State = EState::Dropped; + item.WaitTxId = InvalidTxId; + Self->PersistExportItemState(db, exportInfo, itemIdx); + + if (exportInfo->AllItemsAreDropped()) { + AllocateTxId(exportInfo); + } + } else { + SendNotificationsIfFinished(exportInfo, true); // for tests + + if (exportInfo->Uid) { + Self->ExportsByUid.erase(exportInfo->Uid); + } + + Self->Exports.erase(exportInfo->Id); + Self->PersistRemoveExport(db, exportInfo); + } + return; + + default: return SendNotificationsIfFinished(exportInfo); - } - - Self->PersistExportState(db, exportInfo); - SendNotificationsIfFinished(exportInfo); - } - -}; // TTxProgress - + } + + Self->PersistExportState(db, exportInfo); + SendNotificationsIfFinished(exportInfo); + } + +}; // TTxProgress + ITransaction* TSchemeShard::CreateTxCreateExport(TEvExport::TEvCreateExportRequest::TPtr& ev) { - return new TExport::TTxCreate(this, ev); -} - + return new TExport::TTxCreate(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressExport(ui64 id) { - return new TExport::TTxProgress(this, id); -} - + return new TExport::TTxProgress(this, id); +} + ITransaction* TSchemeShard::CreateTxProgressExport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) { - return new TExport::TTxProgress(this, ev); -} - + return new TExport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressExport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) { - return new TExport::TTxProgress(this, ev); -} - + return new TExport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressExport(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) { - return new TExport::TTxProgress(this, ev); -} - + return new TExport::TTxProgress(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export__forget.cpp b/ydb/core/tx/schemeshard/schemeshard_export__forget.cpp index f196cb9104b..cef8f9f300d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__forget.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__forget.cpp @@ -1,118 +1,118 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_export_flow_proposals.h" -#include "schemeshard_export_helpers.h" -#include "schemeshard_export.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_export_flow_proposals.h" +#include "schemeshard_export_helpers.h" +#include "schemeshard_export.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/ptr.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TExport::TTxForget: public TSchemeShard::TXxport::TTxBase { - TEvExport::TEvForgetExportRequest::TPtr Request; - bool Progress; - - explicit TTxForget(TSelf *self, TEvExport::TEvForgetExportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - , Progress(false) - { - } - - TTxType GetTxType() const override { - return TXTYPE_FORGET_EXPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - auto response = MakeHolder<TEvExport::TEvForgetExportResponse>(request.GetTxId()); - auto& forget = *response->Record.MutableResponse(); - - auto it = Self->Exports.find(request.GetRequest().GetId()); - if (it == Self->Exports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { - forget.SetStatus(Ydb::StatusIds::NOT_FOUND); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - TExportInfo::TPtr exportInfo = it->second; - - if (!exportInfo->IsValid()) { - forget.SetStatus(Ydb::StatusIds::UNDETERMINED); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - if (exportInfo->IsInProgress()) { - forget.SetStatus(Ydb::StatusIds::PRECONDITION_FAILED); - AddIssue(forget, "Export operation is in progress"); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - Y_VERIFY(exportInfo->IsFinished()); - NIceDb::TNiceDb db(txc.DB); - - const TPath exportPath = TPath::Init(exportInfo->ExportPathId, Self); - if (!exportPath.IsResolved()) { - SendNotificationsIfFinished(exportInfo, true); // for tests - - if (exportInfo->Uid) { - Self->ExportsByUid.erase(exportInfo->Uid); - } - - Self->Exports.erase(exportInfo->Id); - Self->PersistRemoveExport(db, exportInfo); - } else { - exportInfo->WaitTxId = InvalidTxId; - exportInfo->State = TExportInfo::EState::Dropping; - Self->PersistExportState(db, exportInfo); - - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - auto& item = exportInfo->Items.at(itemIdx); - - item.WaitTxId = InvalidTxId; - item.State = TExportInfo::EState::Dropped; - - const TPath itemPath = TPath::Resolve(ExportItemPathName(Self, exportInfo, itemIdx), Self); - if (itemPath.IsResolved() && !itemPath.IsDeleted()) { - item.State = TExportInfo::EState::Dropping; - } - - Self->PersistExportItemState(db, exportInfo, itemIdx); - } - - Progress = true; - } - - forget.SetStatus(Ydb::StatusIds::SUCCESS); - Send(Request->Sender, std::move(response), 0, Request->Cookie); - - return true; - } - - void DoComplete(const TActorContext& ctx) override { - if (Progress) { - const ui64 id = Request->Get()->Record.GetRequest().GetId(); - Self->Execute(Self->CreateTxProgressExport(id), ctx); - } - } - -}; // TTxForget - + TEvExport::TEvForgetExportRequest::TPtr Request; + bool Progress; + + explicit TTxForget(TSelf *self, TEvExport::TEvForgetExportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + , Progress(false) + { + } + + TTxType GetTxType() const override { + return TXTYPE_FORGET_EXPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + auto response = MakeHolder<TEvExport::TEvForgetExportResponse>(request.GetTxId()); + auto& forget = *response->Record.MutableResponse(); + + auto it = Self->Exports.find(request.GetRequest().GetId()); + if (it == Self->Exports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { + forget.SetStatus(Ydb::StatusIds::NOT_FOUND); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + TExportInfo::TPtr exportInfo = it->second; + + if (!exportInfo->IsValid()) { + forget.SetStatus(Ydb::StatusIds::UNDETERMINED); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + if (exportInfo->IsInProgress()) { + forget.SetStatus(Ydb::StatusIds::PRECONDITION_FAILED); + AddIssue(forget, "Export operation is in progress"); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + Y_VERIFY(exportInfo->IsFinished()); + NIceDb::TNiceDb db(txc.DB); + + const TPath exportPath = TPath::Init(exportInfo->ExportPathId, Self); + if (!exportPath.IsResolved()) { + SendNotificationsIfFinished(exportInfo, true); // for tests + + if (exportInfo->Uid) { + Self->ExportsByUid.erase(exportInfo->Uid); + } + + Self->Exports.erase(exportInfo->Id); + Self->PersistRemoveExport(db, exportInfo); + } else { + exportInfo->WaitTxId = InvalidTxId; + exportInfo->State = TExportInfo::EState::Dropping; + Self->PersistExportState(db, exportInfo); + + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + auto& item = exportInfo->Items.at(itemIdx); + + item.WaitTxId = InvalidTxId; + item.State = TExportInfo::EState::Dropped; + + const TPath itemPath = TPath::Resolve(ExportItemPathName(Self, exportInfo, itemIdx), Self); + if (itemPath.IsResolved() && !itemPath.IsDeleted()) { + item.State = TExportInfo::EState::Dropping; + } + + Self->PersistExportItemState(db, exportInfo, itemIdx); + } + + Progress = true; + } + + forget.SetStatus(Ydb::StatusIds::SUCCESS); + Send(Request->Sender, std::move(response), 0, Request->Cookie); + + return true; + } + + void DoComplete(const TActorContext& ctx) override { + if (Progress) { + const ui64 id = Request->Get()->Record.GetRequest().GetId(); + Self->Execute(Self->CreateTxProgressExport(id), ctx); + } + } + +}; // TTxForget + ITransaction* TSchemeShard::CreateTxForgetExport(TEvExport::TEvForgetExportRequest::TPtr& ev) { - return new TExport::TTxForget(this, ev); -} - + return new TExport::TTxForget(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export__get.cpp b/ydb/core/tx/schemeshard/schemeshard_export__get.cpp index 900312e544b..c90249bb174 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__get.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__get.cpp @@ -1,31 +1,31 @@ -#include "schemeshard_xxport__get.h" -#include "schemeshard_export.h" - -namespace NKikimr { +#include "schemeshard_xxport__get.h" +#include "schemeshard_export.h" + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TExport::TTxGet: public TSchemeShard::TXxport::TTxGet< - TExportInfo, - TEvExport::TEvGetExportRequest, - TEvExport::TEvGetExportResponse -> { - using TTxGetBase::TTxGetBase; - - TTxType GetTxType() const override { - return TXTYPE_GET_EXPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - return DoExecuteImpl(Self->Exports, txc, ctx); - } - -}; // TTxGet - + TExportInfo, + TEvExport::TEvGetExportRequest, + TEvExport::TEvGetExportResponse +> { + using TTxGetBase::TTxGetBase; + + TTxType GetTxType() const override { + return TXTYPE_GET_EXPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + return DoExecuteImpl(Self->Exports, txc, ctx); + } + +}; // TTxGet + ITransaction* TSchemeShard::CreateTxGetExport(TEvExport::TEvGetExportRequest::TPtr& ev) { - return new TExport::TTxGet(this, ev); -} - + return new TExport::TTxGet(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export__list.cpp b/ydb/core/tx/schemeshard/schemeshard_export__list.cpp index 032a632d542..595f7d7090f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__list.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__list.cpp @@ -1,42 +1,42 @@ -#include "schemeshard_xxport__list.h" -#include "schemeshard_export.h" - -namespace NKikimr { +#include "schemeshard_xxport__list.h" +#include "schemeshard_export.h" + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TExport::TTxList: public TSchemeShard::TXxport::TTxList< - TExportInfo, - TEvExport::TEvListExportsRequest, - TEvExport::TEvListExportsResponse, + TExportInfo, + TEvExport::TEvListExportsRequest, + TEvExport::TEvListExportsResponse, TSchemeShard::TExport::TTxList -> { - using TTxListBase::TTxListBase; - - TTxType GetTxType() const override { - return TXTYPE_LIST_EXPORTS; - } - - static bool TryParseKind(const TString& str, TExportInfo::EKind& parsed) { - if (str == "export/s3") { - parsed = TExportInfo::EKind::S3; - } else { // fallback to yt - parsed = TExportInfo::EKind::YT; - } - - return true; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - return DoExecuteImpl(Self->Exports, txc, ctx); - } - -}; // TTxList - +> { + using TTxListBase::TTxListBase; + + TTxType GetTxType() const override { + return TXTYPE_LIST_EXPORTS; + } + + static bool TryParseKind(const TString& str, TExportInfo::EKind& parsed) { + if (str == "export/s3") { + parsed = TExportInfo::EKind::S3; + } else { // fallback to yt + parsed = TExportInfo::EKind::YT; + } + + return true; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + return DoExecuteImpl(Self->Exports, txc, ctx); + } + +}; // TTxList + ITransaction* TSchemeShard::CreateTxListExports(TEvExport::TEvListExportsRequest::TPtr& ev) { - return new TExport::TTxList(this, ev); -} - + return new TExport::TTxList(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp index a2e7be1edf6..17223ef5a59 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp @@ -1,221 +1,221 @@ -#include "schemeshard_export_flow_proposals.h" -#include "schemeshard_path_describer.h" - -#include <util/string/builder.h> -#include <util/string/cast.h> - -namespace NKikimr { +#include "schemeshard_export_flow_proposals.h" +#include "schemeshard_path_describer.h" + +#include <util/string/builder.h> +#include <util/string/cast.h> + +namespace NKikimr { namespace NSchemeShard { - + THolder<TEvSchemeShard::TEvModifySchemeTransaction> MkDirPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -) { + TTxId txId, + const TExportInfo::TPtr exportInfo +) { auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - auto& record = propose->Record; - - if (exportInfo->UserSID) { - record.SetOwner(*exportInfo->UserSID); - } - + auto& record = propose->Record; + + if (exportInfo->UserSID) { + record.SetOwner(*exportInfo->UserSID); + } + auto& modifyScheme = *record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpMkDir); modifyScheme.SetInternal(true); - - const TPath domainPath = TPath::Init(exportInfo->DomainPathId, ss); - modifyScheme.SetWorkingDir(domainPath.PathString()); - - auto& mkDir = *modifyScheme.MutableMkDir(); - mkDir.SetName(Sprintf("export-%" PRIu64, exportInfo->Id)); - - return propose; -} - + + const TPath domainPath = TPath::Init(exportInfo->DomainPathId, ss); + modifyScheme.SetWorkingDir(domainPath.PathString()); + + auto& mkDir = *modifyScheme.MutableMkDir(); + mkDir.SetName(Sprintf("export-%" PRIu64, exportInfo->Id)); + + return propose; +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CopyTablesPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -) { + TTxId txId, + const TExportInfo::TPtr exportInfo +) { auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - auto& record = propose->Record; - - if (exportInfo->UserSID) { - record.SetOwner(*exportInfo->UserSID); - } - + auto& record = propose->Record; + + if (exportInfo->UserSID) { + record.SetOwner(*exportInfo->UserSID); + } + auto& modifyScheme = *record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateConsistentCopyTables); modifyScheme.SetInternal(true); - - auto& copyTables = *modifyScheme.MutableCreateConsistentCopyTables()->MutableCopyTableDescriptions(); - copyTables.Reserve(exportInfo->Items.size()); - - const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); - const TString& exportPathName = exportPath.PathString(); - - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - const auto& item = exportInfo->Items.at(itemIdx); - - auto& desc = *copyTables.Add(); - desc.SetSrcPath(item.SourcePathName); - desc.SetDstPath(ExportItemPathName(exportPathName, itemIdx)); - desc.SetOmitIndexes(true); + + auto& copyTables = *modifyScheme.MutableCreateConsistentCopyTables()->MutableCopyTableDescriptions(); + copyTables.Reserve(exportInfo->Items.size()); + + const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); + const TString& exportPathName = exportPath.PathString(); + + for (ui32 itemIdx : xrange(exportInfo->Items.size())) { + const auto& item = exportInfo->Items.at(itemIdx); + + auto& desc = *copyTables.Add(); + desc.SetSrcPath(item.SourcePathName); + desc.SetDstPath(ExportItemPathName(exportPathName, itemIdx)); + desc.SetOmitIndexes(true); desc.SetOmitFollowers(true); - desc.SetIsBackup(true); - } - - return propose; -} - + desc.SetIsBackup(true); + } + + return propose; +} + static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, const TPathId& pathId) { NKikimrSchemeOp::TDescribeOptions opts; - opts.SetReturnPartitioningInfo(false); - opts.SetReturnPartitionConfig(true); - opts.SetReturnBoundaries(true); - - auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts); - auto record = desc->GetRecord(); - - return record.GetPathDescription(); -} - + opts.SetReturnPartitioningInfo(false); + opts.SetReturnPartitionConfig(true); + opts.SetReturnBoundaries(true); + + auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts); + auto record = desc->GetRecord(); + + return record.GetPathDescription(); +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> BackupPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo, - ui32 itemIdx -) { - Y_VERIFY(itemIdx < exportInfo->Items.size()); - + TTxId txId, + const TExportInfo::TPtr exportInfo, + ui32 itemIdx +) { + Y_VERIFY(itemIdx < exportInfo->Items.size()); + auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - + auto& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpBackup); modifyScheme.SetInternal(true); - - const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); - const TString& exportPathName = exportPath.PathString(); - modifyScheme.SetWorkingDir(exportPathName); - - auto& task = *modifyScheme.MutableBackup(); - task.SetTableName(ToString(itemIdx)); - task.SetNeedToBill(!exportInfo->UserSID || !ss->SystemBackupSIDs.contains(*exportInfo->UserSID)); - - const TPath sourcePath = TPath::Init(exportInfo->Items[itemIdx].SourcePathId, ss); - if (sourcePath.IsResolved()) { - task.MutableTable()->CopyFrom(GetTableDescription(ss, sourcePath.Base()->PathId)); - } - - switch (exportInfo->Kind) { - case TExportInfo::EKind::YT: - { - Ydb::Export::ExportToYtSettings exportSettings; - Y_VERIFY(exportSettings.ParseFromString(exportInfo->Settings)); - - task.SetNumberOfRetries(exportSettings.number_of_retries()); - auto& backupSettings = *task.MutableYTSettings(); - backupSettings.SetHost(exportSettings.host()); - backupSettings.SetPort(exportSettings.port()); - backupSettings.SetToken(exportSettings.token()); - backupSettings.SetTablePattern(exportSettings.items(itemIdx).destination_path()); - backupSettings.SetUseTypeV3(exportSettings.use_type_v3()); - } - break; - - case TExportInfo::EKind::S3: - { - Ydb::Export::ExportToS3Settings exportSettings; - Y_VERIFY(exportSettings.ParseFromString(exportInfo->Settings)); - - task.SetNumberOfRetries(exportSettings.number_of_retries()); - auto& backupSettings = *task.MutableS3Settings(); - backupSettings.SetEndpoint(exportSettings.endpoint()); - backupSettings.SetBucket(exportSettings.bucket()); - backupSettings.SetAccessKey(exportSettings.access_key()); - backupSettings.SetSecretKey(exportSettings.secret_key()); - backupSettings.SetObjectKeyPattern(exportSettings.items(itemIdx).destination_prefix()); - backupSettings.SetStorageClass(exportSettings.storage_class()); - - switch (exportSettings.scheme()) { - case Ydb::Export::ExportToS3Settings::HTTP: + + const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); + const TString& exportPathName = exportPath.PathString(); + modifyScheme.SetWorkingDir(exportPathName); + + auto& task = *modifyScheme.MutableBackup(); + task.SetTableName(ToString(itemIdx)); + task.SetNeedToBill(!exportInfo->UserSID || !ss->SystemBackupSIDs.contains(*exportInfo->UserSID)); + + const TPath sourcePath = TPath::Init(exportInfo->Items[itemIdx].SourcePathId, ss); + if (sourcePath.IsResolved()) { + task.MutableTable()->CopyFrom(GetTableDescription(ss, sourcePath.Base()->PathId)); + } + + switch (exportInfo->Kind) { + case TExportInfo::EKind::YT: + { + Ydb::Export::ExportToYtSettings exportSettings; + Y_VERIFY(exportSettings.ParseFromString(exportInfo->Settings)); + + task.SetNumberOfRetries(exportSettings.number_of_retries()); + auto& backupSettings = *task.MutableYTSettings(); + backupSettings.SetHost(exportSettings.host()); + backupSettings.SetPort(exportSettings.port()); + backupSettings.SetToken(exportSettings.token()); + backupSettings.SetTablePattern(exportSettings.items(itemIdx).destination_path()); + backupSettings.SetUseTypeV3(exportSettings.use_type_v3()); + } + break; + + case TExportInfo::EKind::S3: + { + Ydb::Export::ExportToS3Settings exportSettings; + Y_VERIFY(exportSettings.ParseFromString(exportInfo->Settings)); + + task.SetNumberOfRetries(exportSettings.number_of_retries()); + auto& backupSettings = *task.MutableS3Settings(); + backupSettings.SetEndpoint(exportSettings.endpoint()); + backupSettings.SetBucket(exportSettings.bucket()); + backupSettings.SetAccessKey(exportSettings.access_key()); + backupSettings.SetSecretKey(exportSettings.secret_key()); + backupSettings.SetObjectKeyPattern(exportSettings.items(itemIdx).destination_prefix()); + backupSettings.SetStorageClass(exportSettings.storage_class()); + + switch (exportSettings.scheme()) { + case Ydb::Export::ExportToS3Settings::HTTP: backupSettings.SetScheme(NKikimrSchemeOp::TS3Settings::HTTP); - break; - case Ydb::Export::ExportToS3Settings::HTTPS: + break; + case Ydb::Export::ExportToS3Settings::HTTPS: backupSettings.SetScheme(NKikimrSchemeOp::TS3Settings::HTTPS); - break; - default: - Y_FAIL("Unknown scheme"); - } - } - break; - } - - return propose; -} - + break; + default: + Y_FAIL("Unknown scheme"); + } + } + break; + } + + return propose; +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> DropPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo, - ui32 itemIdx -) { + TTxId txId, + const TExportInfo::TPtr exportInfo, + ui32 itemIdx +) { auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - + auto& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); modifyScheme.SetInternal(true); - - const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); - modifyScheme.SetWorkingDir(exportPath.PathString()); - - auto& drop = *modifyScheme.MutableDrop(); - drop.SetName(ToString(itemIdx)); - - return propose; -} - + + const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); + modifyScheme.SetWorkingDir(exportPath.PathString()); + + auto& drop = *modifyScheme.MutableDrop(); + drop.SetName(ToString(itemIdx)); + + return propose; +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> DropPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -) { + TTxId txId, + const TExportInfo::TPtr exportInfo +) { auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - + auto& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpRmDir); modifyScheme.SetInternal(true); - - const TPath domainPath = TPath::Init(exportInfo->DomainPathId, ss); - modifyScheme.SetWorkingDir(domainPath.PathString()); - - auto& drop = *modifyScheme.MutableDrop(); - drop.SetName(Sprintf("export-%" PRIu64, exportInfo->Id)); - - return propose; -} - + + const TPath domainPath = TPath::Init(exportInfo->DomainPathId, ss); + modifyScheme.SetWorkingDir(domainPath.PathString()); + + auto& drop = *modifyScheme.MutableDrop(); + drop.SetName(Sprintf("export-%" PRIu64, exportInfo->Id)); + + return propose; +} + THolder<TEvSchemeShard::TEvCancelTx> CancelPropose( - const TExportInfo::TPtr exportInfo, - TTxId backupTxId -) { + const TExportInfo::TPtr exportInfo, + TTxId backupTxId +) { auto propose = MakeHolder<TEvSchemeShard::TEvCancelTx>(); - - auto& record = propose->Record; - record.SetTxId(exportInfo->Id); - record.SetTargetTxId(ui64(backupTxId)); - - return propose; -} - + + auto& record = propose->Record; + record.SetTxId(exportInfo->Id); + record.SetTargetTxId(ui64(backupTxId)); + + return propose; +} + TString ExportItemPathName(TSchemeShard* ss, const TExportInfo::TPtr exportInfo, ui32 itemIdx) { - const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); - return ExportItemPathName(exportPath.PathString(), itemIdx); -} - -TString ExportItemPathName(const TString& exportPathName, ui32 itemIdx) { - return TStringBuilder() << exportPathName << "/" << itemIdx; -} - + const TPath exportPath = TPath::Init(exportInfo->ExportPathId, ss); + return ExportItemPathName(exportPath.PathString(), itemIdx); +} + +TString ExportItemPathName(const TString& exportPathName, ui32 itemIdx) { + return TStringBuilder() << exportPathName << "/" << itemIdx; +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.h b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.h index 1025b63edc7..a84cb925ea0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.h +++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.h @@ -1,51 +1,51 @@ -#pragma once - -#include "schemeshard_impl.h" - -#include <util/generic/ptr.h> - -namespace NKikimr { +#pragma once + +#include "schemeshard_impl.h" + +#include <util/generic/ptr.h> + +namespace NKikimr { namespace NSchemeShard { - + THolder<TEvSchemeShard::TEvModifySchemeTransaction> MkDirPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -); - + TTxId txId, + const TExportInfo::TPtr exportInfo +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CopyTablesPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -); - + TTxId txId, + const TExportInfo::TPtr exportInfo +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> BackupPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo, - ui32 itemIdx -); - + TTxId txId, + const TExportInfo::TPtr exportInfo, + ui32 itemIdx +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> DropPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo, - ui32 itemIdx -); - + TTxId txId, + const TExportInfo::TPtr exportInfo, + ui32 itemIdx +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> DropPropose( TSchemeShard* ss, - TTxId txId, - const TExportInfo::TPtr exportInfo -); - + TTxId txId, + const TExportInfo::TPtr exportInfo +); + THolder<TEvSchemeShard::TEvCancelTx> CancelPropose( - const TExportInfo::TPtr exportInfo, - TTxId backupTxId -); - + const TExportInfo::TPtr exportInfo, + TTxId backupTxId +); + TString ExportItemPathName(TSchemeShard* ss, const TExportInfo::TPtr exportInfo, ui32 itemIdx); -TString ExportItemPathName(const TString& exportPathName, ui32 itemIdx); - +TString ExportItemPathName(const TString& exportPathName, ui32 itemIdx); + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export_helpers.h b/ydb/core/tx/schemeshard/schemeshard_export_helpers.h index 021d9adc94e..9c67c867489 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_helpers.h +++ b/ydb/core/tx/schemeshard/schemeshard_export_helpers.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/protos/issue_severity.pb.h> - -#if defined LOG_T || \ - defined LOG_D || \ - defined LOG_I || \ - defined LOG_N || \ - defined LOG_W || \ - defined LOG_W -#error log macro redefinition -#endif - -#define LOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) -#define LOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) -#define LOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) -#define LOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) -#define LOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) -#define LOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) - -template <typename T> -void AddIssue(T& response, const TString& message, NYql::TSeverityIds::ESeverityId severity = NYql::TSeverityIds::S_ERROR) { - auto& issue = *response.AddIssues(); - issue.set_severity(severity); - issue.set_message(message); -} + +#if defined LOG_T || \ + defined LOG_D || \ + defined LOG_I || \ + defined LOG_N || \ + defined LOG_W || \ + defined LOG_W +#error log macro redefinition +#endif + +#define LOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) +#define LOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) +#define LOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) +#define LOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) +#define LOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) +#define LOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::EXPORT, stream) + +template <typename T> +void AddIssue(T& response, const TString& message, NYql::TSeverityIds::ESeverityId severity = NYql::TSeverityIds::S_ERROR) { + auto& issue = *response.AddIssues(); + issue.set_severity(severity); + issue.set_message(message); +} diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index d5df3a99b20..0715543a229 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -295,15 +295,15 @@ void TSchemeShard::Clear() { TxInFlight.clear(); ShardInfos.clear(); Indexes.clear(); - CdcStreams.clear(); + CdcStreams.clear(); RevertedMigrations.clear(); Sequences.clear(); - Replications.clear(); + Replications.clear(); for(ui32 idx = 0; idx < TabletCounters->Simple().Size(); ++idx) { TabletCounters->Simple()[idx].Set(0); } - TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].Clear(); + TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].Clear(); } void TSchemeShard::IncrementPathDbRefCount(const TPathId& pathId, const TStringBuf& debug) { @@ -1234,9 +1234,9 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxCreateTableIndex: case TTxState::TxCreateOlapStore: case TTxState::TxCreateOlapTable: - case TTxState::TxCreateCdcStream: + case TTxState::TxCreateCdcStream: case TTxState::TxCreateSequence: - case TTxState::TxCreateReplication: + case TTxState::TxCreateReplication: return TPathElement::EPathState::EPathStateCreate; case TTxState::TxAlterPQGroup: case TTxState::TxAlterTable: @@ -1255,12 +1255,12 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxDropTableIndexAtMainTable: case TTxState::TxAlterOlapStore: case TTxState::TxAlterOlapTable: - case TTxState::TxAlterCdcStream: - case TTxState::TxAlterCdcStreamAtTable: - case TTxState::TxCreateCdcStreamAtTable: - case TTxState::TxDropCdcStreamAtTable: + case TTxState::TxAlterCdcStream: + case TTxState::TxAlterCdcStreamAtTable: + case TTxState::TxCreateCdcStreamAtTable: + case TTxState::TxDropCdcStreamAtTable: case TTxState::TxAlterSequence: - case TTxState::TxAlterReplication: + case TTxState::TxAlterReplication: return TPathElement::EPathState::EPathStateAlter; case TTxState::TxDropTable: case TTxState::TxDropPQGroup: @@ -1275,14 +1275,14 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxDropTableIndex: case TTxState::TxDropOlapStore: case TTxState::TxDropOlapTable: - case TTxState::TxDropCdcStream: + case TTxState::TxDropCdcStream: case TTxState::TxDropSequence: - case TTxState::TxDropReplication: + case TTxState::TxDropReplication: return TPathElement::EPathState::EPathStateDrop; case TTxState::TxBackup: return TPathElement::EPathState::EPathStateBackup; - case TTxState::TxRestore: - return TPathElement::EPathState::EPathStateRestore; + case TTxState::TxRestore: + return TPathElement::EPathState::EPathStateRestore; case TTxState::TxUpgradeSubDomain: return TPathElement::EPathState::EPathStateUpgrade; case TTxState::TxUpgradeSubDomainDecision: @@ -1392,67 +1392,67 @@ void TSchemeShard::PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId } void TSchemeShard::PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId) { - Y_VERIFY(PathsById.contains(pathId)); - auto path = PathsById.at(pathId); - - Y_VERIFY(CdcStreams.contains(pathId)); - auto stream = CdcStreams.at(pathId); - - Y_VERIFY(IsLocalId(pathId)); - Y_VERIFY(path->IsCdcStream()); - - auto alterData = stream->AlterData; - Y_VERIFY(alterData); - Y_VERIFY(stream->AlterVersion < alterData->AlterVersion); - - db.Table<Schema::CdcStream>().Key(pathId.OwnerId, pathId.LocalPathId).Update( - NIceDb::TUpdate<Schema::CdcStream::AlterVersion>(alterData->AlterVersion), - NIceDb::TUpdate<Schema::CdcStream::Mode>(alterData->Mode), - NIceDb::TUpdate<Schema::CdcStream::State>(alterData->State) - ); - - db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); -} - + Y_VERIFY(PathsById.contains(pathId)); + auto path = PathsById.at(pathId); + + Y_VERIFY(CdcStreams.contains(pathId)); + auto stream = CdcStreams.at(pathId); + + Y_VERIFY(IsLocalId(pathId)); + Y_VERIFY(path->IsCdcStream()); + + auto alterData = stream->AlterData; + Y_VERIFY(alterData); + Y_VERIFY(stream->AlterVersion < alterData->AlterVersion); + + db.Table<Schema::CdcStream>().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate<Schema::CdcStream::AlterVersion>(alterData->AlterVersion), + NIceDb::TUpdate<Schema::CdcStream::Mode>(alterData->Mode), + NIceDb::TUpdate<Schema::CdcStream::State>(alterData->State) + ); + + db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); +} + void TSchemeShard::PersistCdcStreamAlterData(NIceDb::TNiceDb& db, const TPathId& pathId) { - Y_VERIFY(PathsById.contains(pathId)); - auto path = PathsById.at(pathId); - - Y_VERIFY(CdcStreams.contains(pathId)); - auto stream = CdcStreams.at(pathId); - - Y_VERIFY(IsLocalId(pathId)); - Y_VERIFY(path->IsCdcStream()); - - auto alterData = stream->AlterData; - Y_VERIFY(alterData); - - db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Update( - NIceDb::TUpdate<Schema::CdcStreamAlterData::AlterVersion>(alterData->AlterVersion), - NIceDb::TUpdate<Schema::CdcStreamAlterData::Mode>(alterData->Mode), - NIceDb::TUpdate<Schema::CdcStreamAlterData::State>(alterData->State) - ); -} - + Y_VERIFY(PathsById.contains(pathId)); + auto path = PathsById.at(pathId); + + Y_VERIFY(CdcStreams.contains(pathId)); + auto stream = CdcStreams.at(pathId); + + Y_VERIFY(IsLocalId(pathId)); + Y_VERIFY(path->IsCdcStream()); + + auto alterData = stream->AlterData; + Y_VERIFY(alterData); + + db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate<Schema::CdcStreamAlterData::AlterVersion>(alterData->AlterVersion), + NIceDb::TUpdate<Schema::CdcStreamAlterData::Mode>(alterData->Mode), + NIceDb::TUpdate<Schema::CdcStreamAlterData::State>(alterData->State) + ); +} + void TSchemeShard::PersistRemoveCdcStream(NIceDb::TNiceDb &db, const TPathId& pathId) { - Y_VERIFY(PathsById.contains(pathId)); - auto path = PathsById.at(pathId); - - if (!CdcStreams.contains(pathId)) { - return; - } - - auto stream = CdcStreams.at(pathId); - if (stream->AlterData) { - db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); - } - - db.Table<Schema::CdcStream>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); - - CdcStreams.erase(pathId); - DecrementPathDbRefCount(pathId); -} - + Y_VERIFY(PathsById.contains(pathId)); + auto path = PathsById.at(pathId); + + if (!CdcStreams.contains(pathId)) { + return; + } + + auto stream = CdcStreams.at(pathId); + if (stream->AlterData) { + db.Table<Schema::CdcStreamAlterData>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); + } + + db.Table<Schema::CdcStream>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); + + CdcStreams.erase(pathId); + DecrementPathDbRefCount(pathId); +} + void TSchemeShard::PersistAlterUserAttributes(NIceDb::TNiceDb& db, TPathId pathId) { Y_VERIFY(PathsById.contains(pathId)); TPathElement::TPtr element = PathsById.at(pathId); @@ -1676,10 +1676,10 @@ void TSchemeShard::PersistSubDomainAlter(NIceDb::TNiceDb& db, const TPathId& pat db.Table<Schema::SubDomainsAlterData>().Key(pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::SubDomainsAlterData::AlterVersion>(subDomain.GetVersion()), NIceDb::TUpdate<Schema::SubDomainsAlterData::PlanResolution>(subDomain.GetPlanResolution()), - NIceDb::TUpdate<Schema::SubDomainsAlterData::TimeCastBuckets>(subDomain.GetTCB()), - NIceDb::TUpdate<Schema::SubDomainsAlterData::ResourcesDomainOwnerPathId>(subDomain.GetResourcesDomainId().OwnerId), - NIceDb::TUpdate<Schema::SubDomainsAlterData::ResourcesDomainLocalPathId>(subDomain.GetResourcesDomainId().LocalPathId), - NIceDb::TUpdate<Schema::SubDomainsAlterData::SharedHiveId>(subDomain.GetSharedHive())); + NIceDb::TUpdate<Schema::SubDomainsAlterData::TimeCastBuckets>(subDomain.GetTCB()), + NIceDb::TUpdate<Schema::SubDomainsAlterData::ResourcesDomainOwnerPathId>(subDomain.GetResourcesDomainId().OwnerId), + NIceDb::TUpdate<Schema::SubDomainsAlterData::ResourcesDomainLocalPathId>(subDomain.GetResourcesDomainId().LocalPathId), + NIceDb::TUpdate<Schema::SubDomainsAlterData::SharedHiveId>(subDomain.GetSharedHive())); if (subDomain.GetDeclaredSchemeQuotas()) { TString declaredSchemeQuotas; @@ -1745,11 +1745,11 @@ void TSchemeShard::PersistSubDomain(NIceDb::TNiceDb& db, const TPathId& pathId, db.Table<Schema::SubDomains>().Key(pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::SubDomains::AlterVersion>(subDomain.GetVersion()), NIceDb::TUpdate<Schema::SubDomains::PlanResolution>(subDomain.GetPlanResolution()), - NIceDb::TUpdate<Schema::SubDomains::TimeCastBuckets>(subDomain.GetTCB()), - NIceDb::TUpdate<Schema::SubDomains::ResourcesDomainOwnerPathId>(subDomain.GetResourcesDomainId().OwnerId), - NIceDb::TUpdate<Schema::SubDomains::ResourcesDomainLocalPathId>(subDomain.GetResourcesDomainId().LocalPathId), - NIceDb::TUpdate<Schema::SubDomains::SharedHiveId>(subDomain.GetSharedHive())); - + NIceDb::TUpdate<Schema::SubDomains::TimeCastBuckets>(subDomain.GetTCB()), + NIceDb::TUpdate<Schema::SubDomains::ResourcesDomainOwnerPathId>(subDomain.GetResourcesDomainId().OwnerId), + NIceDb::TUpdate<Schema::SubDomains::ResourcesDomainLocalPathId>(subDomain.GetResourcesDomainId().LocalPathId), + NIceDb::TUpdate<Schema::SubDomains::SharedHiveId>(subDomain.GetSharedHive())); + PersistSubDomainDeclaredSchemeQuotas(db, pathId, subDomain); PersistSubDomainDatabaseQuotas(db, pathId, subDomain); PersistSubDomainState(db, pathId, subDomain); @@ -1882,9 +1882,9 @@ void TSchemeShard::PersistCreateTxId(NIceDb::TNiceDb& db, const TPathId pathId, Y_VERIFY(IsLocalId(pathId)); db.Table<Schema::Paths>().Key(pathId.LocalPathId).Update( - NIceDb::TUpdate<Schema::Paths::CreateTxId>(txId)); -} - + NIceDb::TUpdate<Schema::Paths::CreateTxId>(txId)); +} + void TSchemeShard::PersistCreateStep(NIceDb::TNiceDb& db, const TPathId pathId, TStepId step) { Y_VERIFY(IsLocalId(pathId)); @@ -1992,10 +1992,10 @@ void TSchemeShard::ChangeTxState(NIceDb::TNiceDb& db, const TOperationId opId, T } void TSchemeShard::PersistCancelTx(NIceDb::TNiceDb &db, const TOperationId opId, const TTxState &txState) { - Y_VERIFY(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); + Y_VERIFY(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); db.Table<Schema::TxInFlightV2>().Key(opId.GetTxId(), opId.GetSubTxId()).Update( - NIceDb::TUpdate<Schema::TxInFlightV2::CancelBackup>(txState.Cancel)); + NIceDb::TUpdate<Schema::TxInFlightV2::CancelBackup>(txState.Cancel)); } void TSchemeShard::PersistTxPlanStep(NIceDb::TNiceDb &db, TOperationId opId, TStepId step) { @@ -2040,16 +2040,16 @@ void TSchemeShard::PersistTablePartitioning(NIceDb::TNiceDb& db, const TPathId p Y_VERIFY(IsLocalId(partition.ShardIdx)); db.Table<Schema::TablePartitions>().Key(pathId.LocalPathId, pi).Update( NIceDb::TUpdate<Schema::TablePartitions::RangeEnd>(partition.EndOfRange), - NIceDb::TUpdate<Schema::TablePartitions::DatashardIdx>(partition.ShardIdx.GetLocalId()), - NIceDb::TUpdate<Schema::TablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), - NIceDb::TUpdate<Schema::TablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); + NIceDb::TUpdate<Schema::TablePartitions::DatashardIdx>(partition.ShardIdx.GetLocalId()), + NIceDb::TUpdate<Schema::TablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), + NIceDb::TUpdate<Schema::TablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); } else { db.Table<Schema::MigratedTablePartitions>().Key(pathId.OwnerId, pathId.LocalPathId, pi).Update( NIceDb::TUpdate<Schema::MigratedTablePartitions::RangeEnd>(partition.EndOfRange), NIceDb::TUpdate<Schema::MigratedTablePartitions::OwnerShardIdx>(partition.ShardIdx.GetOwnerId()), - NIceDb::TUpdate<Schema::MigratedTablePartitions::LocalShardIdx>(partition.ShardIdx.GetLocalId()), - NIceDb::TUpdate<Schema::MigratedTablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), - NIceDb::TUpdate<Schema::MigratedTablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); + NIceDb::TUpdate<Schema::MigratedTablePartitions::LocalShardIdx>(partition.ShardIdx.GetLocalId()), + NIceDb::TUpdate<Schema::MigratedTablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), + NIceDb::TUpdate<Schema::MigratedTablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); } } if (IsLocalId(pathId)) { @@ -2068,95 +2068,95 @@ void TSchemeShard::DeleteTablePartitioning(NIceDb::TNiceDb& db, const TPathId pa db.Table<Schema::TablePartitions>().Key(pathId.LocalPathId, pi).Delete(); } db.Table<Schema::MigratedTablePartitions>().Key(pathId.OwnerId, pathId.LocalPathId, pi).Delete(); - db.Table<Schema::TablePartitionStats>().Key(pathId.OwnerId, pathId.LocalPathId, pi).Delete(); + db.Table<Schema::TablePartitionStats>().Key(pathId.OwnerId, pathId.LocalPathId, pi).Delete(); CompactionQueue->Remove(TShardCompactionInfo(partitions[pi].ShardIdx)); } } void TSchemeShard::PersistTablePartitionCondErase(NIceDb::TNiceDb& db, const TPathId& pathId, ui64 id, const TTableInfo::TPtr tableInfo) { - const auto& partition = tableInfo->GetPartitions()[id]; - - if (IsLocalId(pathId)) { - Y_VERIFY(IsLocalId(partition.ShardIdx)); - db.Table<Schema::TablePartitions>().Key(pathId.LocalPathId, id).Update( - NIceDb::TUpdate<Schema::TablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), - NIceDb::TUpdate<Schema::TablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); - } else { - db.Table<Schema::MigratedTablePartitions>().Key(pathId.OwnerId, pathId.LocalPathId, id).Update( - NIceDb::TUpdate<Schema::MigratedTablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), - NIceDb::TUpdate<Schema::MigratedTablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); - } -} - + const auto& partition = tableInfo->GetPartitions()[id]; + + if (IsLocalId(pathId)) { + Y_VERIFY(IsLocalId(partition.ShardIdx)); + db.Table<Schema::TablePartitions>().Key(pathId.LocalPathId, id).Update( + NIceDb::TUpdate<Schema::TablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), + NIceDb::TUpdate<Schema::TablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); + } else { + db.Table<Schema::MigratedTablePartitions>().Key(pathId.OwnerId, pathId.LocalPathId, id).Update( + NIceDb::TUpdate<Schema::MigratedTablePartitions::LastCondErase>(partition.LastCondErase.GetValue()), + NIceDb::TUpdate<Schema::MigratedTablePartitions::NextCondErase>(partition.NextCondErase.GetValue())); + } +} + void TSchemeShard::PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, ui64 partitionId, const TTableInfo::TPartitionStats& stats) { - if (!AppData()->FeatureFlags.GetEnablePersistentPartitionStats()) { - return; - } - - db.Table<Schema::TablePartitionStats>().Key(tableId.OwnerId, tableId.LocalPathId, partitionId).Update( - NIceDb::TUpdate<Schema::TablePartitionStats::SeqNoGeneration>(stats.SeqNo.Generation), - NIceDb::TUpdate<Schema::TablePartitionStats::SeqNoRound>(stats.SeqNo.Round), - - NIceDb::TUpdate<Schema::TablePartitionStats::RowCount>(stats.RowCount), - NIceDb::TUpdate<Schema::TablePartitionStats::DataSize>(stats.DataSize), - NIceDb::TUpdate<Schema::TablePartitionStats::IndexSize>(stats.IndexSize), - - NIceDb::TUpdate<Schema::TablePartitionStats::LastAccessTime>(stats.LastAccessTime.GetValue()), - NIceDb::TUpdate<Schema::TablePartitionStats::LastUpdateTime>(stats.LastUpdateTime.GetValue()), - - NIceDb::TUpdate<Schema::TablePartitionStats::ImmediateTxCompleted>(stats.ImmediateTxCompleted), - NIceDb::TUpdate<Schema::TablePartitionStats::PlannedTxCompleted>(stats.PlannedTxCompleted), - NIceDb::TUpdate<Schema::TablePartitionStats::TxRejectedByOverload>(stats.TxRejectedByOverload), - NIceDb::TUpdate<Schema::TablePartitionStats::TxRejectedBySpace>(stats.TxRejectedBySpace), - NIceDb::TUpdate<Schema::TablePartitionStats::TxCompleteLag>(stats.TxCompleteLag.GetValue()), - NIceDb::TUpdate<Schema::TablePartitionStats::InFlightTxCount>(stats.InFlightTxCount), - - NIceDb::TUpdate<Schema::TablePartitionStats::RowUpdates>(stats.RowUpdates), - NIceDb::TUpdate<Schema::TablePartitionStats::RowDeletes>(stats.RowDeletes), - NIceDb::TUpdate<Schema::TablePartitionStats::RowReads>(stats.RowReads), - NIceDb::TUpdate<Schema::TablePartitionStats::RangeReads>(stats.RangeReads), - NIceDb::TUpdate<Schema::TablePartitionStats::RangeReadRows>(stats.RangeReadRows), - + if (!AppData()->FeatureFlags.GetEnablePersistentPartitionStats()) { + return; + } + + db.Table<Schema::TablePartitionStats>().Key(tableId.OwnerId, tableId.LocalPathId, partitionId).Update( + NIceDb::TUpdate<Schema::TablePartitionStats::SeqNoGeneration>(stats.SeqNo.Generation), + NIceDb::TUpdate<Schema::TablePartitionStats::SeqNoRound>(stats.SeqNo.Round), + + NIceDb::TUpdate<Schema::TablePartitionStats::RowCount>(stats.RowCount), + NIceDb::TUpdate<Schema::TablePartitionStats::DataSize>(stats.DataSize), + NIceDb::TUpdate<Schema::TablePartitionStats::IndexSize>(stats.IndexSize), + + NIceDb::TUpdate<Schema::TablePartitionStats::LastAccessTime>(stats.LastAccessTime.GetValue()), + NIceDb::TUpdate<Schema::TablePartitionStats::LastUpdateTime>(stats.LastUpdateTime.GetValue()), + + NIceDb::TUpdate<Schema::TablePartitionStats::ImmediateTxCompleted>(stats.ImmediateTxCompleted), + NIceDb::TUpdate<Schema::TablePartitionStats::PlannedTxCompleted>(stats.PlannedTxCompleted), + NIceDb::TUpdate<Schema::TablePartitionStats::TxRejectedByOverload>(stats.TxRejectedByOverload), + NIceDb::TUpdate<Schema::TablePartitionStats::TxRejectedBySpace>(stats.TxRejectedBySpace), + NIceDb::TUpdate<Schema::TablePartitionStats::TxCompleteLag>(stats.TxCompleteLag.GetValue()), + NIceDb::TUpdate<Schema::TablePartitionStats::InFlightTxCount>(stats.InFlightTxCount), + + NIceDb::TUpdate<Schema::TablePartitionStats::RowUpdates>(stats.RowUpdates), + NIceDb::TUpdate<Schema::TablePartitionStats::RowDeletes>(stats.RowDeletes), + NIceDb::TUpdate<Schema::TablePartitionStats::RowReads>(stats.RowReads), + NIceDb::TUpdate<Schema::TablePartitionStats::RangeReads>(stats.RangeReads), + NIceDb::TUpdate<Schema::TablePartitionStats::RangeReadRows>(stats.RangeReadRows), + NIceDb::TUpdate<Schema::TablePartitionStats::CPU>(stats.GetCurrentRawCpuUsage()), - NIceDb::TUpdate<Schema::TablePartitionStats::Memory>(stats.Memory), - NIceDb::TUpdate<Schema::TablePartitionStats::Network>(stats.Network), - NIceDb::TUpdate<Schema::TablePartitionStats::Storage>(stats.Storage), - NIceDb::TUpdate<Schema::TablePartitionStats::ReadThroughput>(stats.ReadThroughput), - NIceDb::TUpdate<Schema::TablePartitionStats::WriteThroughput>(stats.WriteThroughput), - NIceDb::TUpdate<Schema::TablePartitionStats::ReadIops>(stats.ReadIops), - NIceDb::TUpdate<Schema::TablePartitionStats::WriteIops>(stats.WriteIops) - ); -} - + NIceDb::TUpdate<Schema::TablePartitionStats::Memory>(stats.Memory), + NIceDb::TUpdate<Schema::TablePartitionStats::Network>(stats.Network), + NIceDb::TUpdate<Schema::TablePartitionStats::Storage>(stats.Storage), + NIceDb::TUpdate<Schema::TablePartitionStats::ReadThroughput>(stats.ReadThroughput), + NIceDb::TUpdate<Schema::TablePartitionStats::WriteThroughput>(stats.WriteThroughput), + NIceDb::TUpdate<Schema::TablePartitionStats::ReadIops>(stats.ReadIops), + NIceDb::TUpdate<Schema::TablePartitionStats::WriteIops>(stats.WriteIops) + ); +} + void TSchemeShard::PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TShardIdx& shardIdx, const TTableInfo::TPtr tableInfo) { - const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); - if (!shardToPartition.contains(shardIdx)) { - return; - } - - const auto& tableStats = tableInfo->GetStats(); - if (!tableStats.PartitionStats.contains(shardIdx)) { - return; - } - - const ui64 partitionId = shardToPartition.at(shardIdx); - const auto& stats = tableStats.PartitionStats.at(shardIdx); - PersistTablePartitionStats(db, tableId, partitionId, stats); -} - + const auto& shardToPartition = tableInfo->GetShard2PartitionIdx(); + if (!shardToPartition.contains(shardIdx)) { + return; + } + + const auto& tableStats = tableInfo->GetStats(); + if (!tableStats.PartitionStats.contains(shardIdx)) { + return; + } + + const ui64 partitionId = shardToPartition.at(shardIdx); + const auto& stats = tableStats.PartitionStats.at(shardIdx); + PersistTablePartitionStats(db, tableId, partitionId, stats); +} + void TSchemeShard::PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TTableInfo::TPtr tableInfo) { - const auto& tableStats = tableInfo->GetStats(); - - for (const auto& [shardIdx, pi] : tableInfo->GetShard2PartitionIdx()) { - if (!tableStats.PartitionStats.contains(shardIdx)) { - continue; - } - - PersistTablePartitionStats(db, tableId, pi, tableStats.PartitionStats.at(shardIdx)); - } -} - + const auto& tableStats = tableInfo->GetStats(); + + for (const auto& [shardIdx, pi] : tableInfo->GetShard2PartitionIdx()) { + if (!tableStats.PartitionStats.contains(shardIdx)) { + continue; + } + + PersistTablePartitionStats(db, tableId, pi, tableStats.PartitionStats.at(shardIdx)); + } +} + void TSchemeShard::PersistTableAlterVersion(NIceDb::TNiceDb& db, const TPathId pathId, const TTableInfo::TPtr tableInfo) { if (pathId.OwnerId == TabletID()) { db.Table<Schema::Tables>().Key(pathId.LocalPathId).Update( @@ -2171,29 +2171,29 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId TString partitionConfig; Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->PartitionConfig().SerializeToString(&partitionConfig); - TString ttlSettings; - if (tableInfo->HasTTLSettings()) { + TString ttlSettings; + if (tableInfo->HasTTLSettings()) { Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->TTLSettings().SerializeToString(&ttlSettings); - } - + } + if (pathId.OwnerId == TabletID()) { db.Table<Schema::Tables>().Key(pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::Tables::NextColId>(tableInfo->NextColumnId), NIceDb::TUpdate<Schema::Tables::PartitionConfig>(partitionConfig), NIceDb::TUpdate<Schema::Tables::AlterVersion>(tableInfo->AlterVersion), NIceDb::TUpdate<Schema::Tables::AlterTable>(TString()), - NIceDb::TUpdate<Schema::Tables::AlterTableFull>(TString()), - NIceDb::TUpdate<Schema::Tables::TTLSettings>(ttlSettings), - NIceDb::TUpdate<Schema::Tables::IsBackup>(tableInfo->IsBackup)); + NIceDb::TUpdate<Schema::Tables::AlterTableFull>(TString()), + NIceDb::TUpdate<Schema::Tables::TTLSettings>(ttlSettings), + NIceDb::TUpdate<Schema::Tables::IsBackup>(tableInfo->IsBackup)); } else { db.Table<Schema::MigratedTables>().Key(pathId.OwnerId, pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::MigratedTables::NextColId>(tableInfo->NextColumnId), NIceDb::TUpdate<Schema::MigratedTables::PartitionConfig>(partitionConfig), NIceDb::TUpdate<Schema::MigratedTables::AlterVersion>(tableInfo->AlterVersion), NIceDb::TUpdate<Schema::MigratedTables::AlterTable>(TString()), - NIceDb::TUpdate<Schema::MigratedTables::AlterTableFull>(TString()), - NIceDb::TUpdate<Schema::MigratedTables::TTLSettings>(ttlSettings), - NIceDb::TUpdate<Schema::MigratedTables::IsBackup>(tableInfo->IsBackup)); + NIceDb::TUpdate<Schema::MigratedTables::AlterTableFull>(TString()), + NIceDb::TUpdate<Schema::MigratedTables::TTLSettings>(ttlSettings), + NIceDb::TUpdate<Schema::MigratedTables::IsBackup>(tableInfo->IsBackup)); } for (auto col : tableInfo->Columns) { @@ -2319,8 +2319,8 @@ void TSchemeShard::PersistAddPersQueueGroupAlter(NIceDb::TNiceDb& db, TPathId pa NIceDb::TUpdate<Schema::PersQueueGroupAlters::MaxPQPerShard>(alterData->MaxPartsPerTablet), NIceDb::TUpdate<Schema::PersQueueGroupAlters::AlterVersion>(alterData->AlterVersion), NIceDb::TUpdate<Schema::PersQueueGroupAlters::TotalGroupCount>(alterData->TotalGroupCount), - NIceDb::TUpdate<Schema::PersQueueGroupAlters::NextPartitionId>(alterData->NextPartitionId), - NIceDb::TUpdate<Schema::PersQueueGroupAlters::BootstrapConfig>(alterData->BootstrapConfig)); + NIceDb::TUpdate<Schema::PersQueueGroupAlters::NextPartitionId>(alterData->NextPartitionId), + NIceDb::TUpdate<Schema::PersQueueGroupAlters::BootstrapConfig>(alterData->BootstrapConfig)); } void TSchemeShard::PersistRemovePersQueueGroupAlter(NIceDb::TNiceDb& db, TPathId pathId) { @@ -2332,22 +2332,22 @@ void TSchemeShard::PersistRemovePersQueueGroupAlter(NIceDb::TNiceDb& db, TPathId void TSchemeShard::PersistPersQueue(NIceDb::TNiceDb &db, TPathId pathId, TShardIdx shardIdx, const TPQShardInfo::TPersQueueInfo& pqInfo) { Y_VERIFY(IsLocalId(pathId)); - db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( + db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( NIceDb::TUpdate<Schema::PersQueues::ShardIdx>(shardIdx.GetLocalId()), - NIceDb::TUpdate<Schema::PersQueues::GroupId>(pqInfo.GroupId), - NIceDb::TUpdate<Schema::PersQueues::AlterVersion>(pqInfo.AlterVersion)); - - if (pqInfo.KeyRange) { - if (pqInfo.KeyRange->FromBound) { - db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( - NIceDb::TUpdate<Schema::PersQueues::RangeBegin>(*pqInfo.KeyRange->FromBound)); - } - - if (pqInfo.KeyRange->ToBound) { - db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( - NIceDb::TUpdate<Schema::PersQueues::RangeEnd>(*pqInfo.KeyRange->ToBound)); - } - } + NIceDb::TUpdate<Schema::PersQueues::GroupId>(pqInfo.GroupId), + NIceDb::TUpdate<Schema::PersQueues::AlterVersion>(pqInfo.AlterVersion)); + + if (pqInfo.KeyRange) { + if (pqInfo.KeyRange->FromBound) { + db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( + NIceDb::TUpdate<Schema::PersQueues::RangeBegin>(*pqInfo.KeyRange->FromBound)); + } + + if (pqInfo.KeyRange->ToBound) { + db.Table<Schema::PersQueues>().Key(pathId.LocalPathId, pqInfo.PqId).Update( + NIceDb::TUpdate<Schema::PersQueues::RangeEnd>(*pqInfo.KeyRange->ToBound)); + } + } } void TSchemeShard::PersistRemovePersQueue(NIceDb::TNiceDb &db, TPathId pathId, ui32 pqId) { @@ -2605,79 +2605,79 @@ void TSchemeShard::PersistUnknownShardDeleted(NIceDb::TNiceDb& db, TShardIdx sha } void TSchemeShard::PersistTxShardStatus(NIceDb::TNiceDb& db, TOperationId opId, TShardIdx shardIdx, const TTxState::TShardStatus& status) { - db.Table<Schema::TxShardStatus>() - .Key(opId.GetTxId(), shardIdx.GetOwnerId(), shardIdx.GetLocalId()) - .Update( - NIceDb::TUpdate<Schema::TxShardStatus::Success>(status.Success), - NIceDb::TUpdate<Schema::TxShardStatus::Error>(status.Error), - NIceDb::TUpdate<Schema::TxShardStatus::BytesProcessed>(status.BytesProcessed), - NIceDb::TUpdate<Schema::TxShardStatus::RowsProcessed>(status.RowsProcessed) - ); -} - + db.Table<Schema::TxShardStatus>() + .Key(opId.GetTxId(), shardIdx.GetOwnerId(), shardIdx.GetLocalId()) + .Update( + NIceDb::TUpdate<Schema::TxShardStatus::Success>(status.Success), + NIceDb::TUpdate<Schema::TxShardStatus::Error>(status.Error), + NIceDb::TUpdate<Schema::TxShardStatus::BytesProcessed>(status.BytesProcessed), + NIceDb::TUpdate<Schema::TxShardStatus::RowsProcessed>(status.RowsProcessed) + ); +} + void TSchemeShard::PersistBackupSettings( NIceDb::TNiceDb& db, TPathId pathId, const NKikimrSchemeOp::TBackupTask& settings) { -#define PERSIST_BACKUP_SETTINGS(Kind) \ - if (settings.Has##Kind()) { \ - if (IsLocalId(pathId)) { \ - db.Table<Schema::BackupSettings>().Key(pathId.LocalPathId).Update( \ - NIceDb::TUpdate<Schema::BackupSettings::TableName>(settings.GetTableName()), \ - NIceDb::TUpdate<Schema::BackupSettings::Kind>(settings.Get##Kind().SerializeAsString()), \ - NIceDb::TUpdate<Schema::BackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \ - NIceDb::TUpdate<Schema::BackupSettings::NeedToBill>(settings.GetNeedToBill()), \ - NIceDb::TUpdate<Schema::BackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \ - NIceDb::TUpdate<Schema::BackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \ - } else { \ - db.Table<Schema::MigratedBackupSettings>().Key(pathId.OwnerId, pathId.LocalPathId).Update( \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::TableName>(settings.GetTableName()), \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::Kind>(settings.Get##Kind().SerializeAsString()), \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::NeedToBill>(settings.GetNeedToBill()), \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \ - NIceDb::TUpdate<Schema::MigratedBackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \ - } \ - } - - PERSIST_BACKUP_SETTINGS(YTSettings) - PERSIST_BACKUP_SETTINGS(S3Settings) - -#undef PERSIST_BACKUP_SETTINGS +#define PERSIST_BACKUP_SETTINGS(Kind) \ + if (settings.Has##Kind()) { \ + if (IsLocalId(pathId)) { \ + db.Table<Schema::BackupSettings>().Key(pathId.LocalPathId).Update( \ + NIceDb::TUpdate<Schema::BackupSettings::TableName>(settings.GetTableName()), \ + NIceDb::TUpdate<Schema::BackupSettings::Kind>(settings.Get##Kind().SerializeAsString()), \ + NIceDb::TUpdate<Schema::BackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \ + NIceDb::TUpdate<Schema::BackupSettings::NeedToBill>(settings.GetNeedToBill()), \ + NIceDb::TUpdate<Schema::BackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \ + NIceDb::TUpdate<Schema::BackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \ + } else { \ + db.Table<Schema::MigratedBackupSettings>().Key(pathId.OwnerId, pathId.LocalPathId).Update( \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::TableName>(settings.GetTableName()), \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::Kind>(settings.Get##Kind().SerializeAsString()), \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::NeedToBill>(settings.GetNeedToBill()), \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \ + NIceDb::TUpdate<Schema::MigratedBackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \ + } \ + } + + PERSIST_BACKUP_SETTINGS(YTSettings) + PERSIST_BACKUP_SETTINGS(S3Settings) + +#undef PERSIST_BACKUP_SETTINGS } void TSchemeShard::PersistCompletedBackupRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& info, TTableInfo::TBackupRestoreResult::EKind kind) { TPathId pathId = txState.TargetPathId; if (IsLocalId(pathId)) { - db.Table<Schema::CompletedBackups>().Key(pathId.LocalPathId, txId, info.CompletionDateTime).Update( - NIceDb::TUpdate<Schema::CompletedBackups::TotalShardCount>(info.TotalShardCount), - NIceDb::TUpdate<Schema::CompletedBackups::SuccessShardCount>(info.SuccessShardCount), - NIceDb::TUpdate<Schema::CompletedBackups::StartTime>(info.StartDateTime), - NIceDb::TUpdate<Schema::CompletedBackups::DataTotalSize>(txState.DataTotalSize), - NIceDb::TUpdate<Schema::CompletedBackups::Kind>(static_cast<ui8>(kind))); + db.Table<Schema::CompletedBackups>().Key(pathId.LocalPathId, txId, info.CompletionDateTime).Update( + NIceDb::TUpdate<Schema::CompletedBackups::TotalShardCount>(info.TotalShardCount), + NIceDb::TUpdate<Schema::CompletedBackups::SuccessShardCount>(info.SuccessShardCount), + NIceDb::TUpdate<Schema::CompletedBackups::StartTime>(info.StartDateTime), + NIceDb::TUpdate<Schema::CompletedBackups::DataTotalSize>(txState.DataTotalSize), + NIceDb::TUpdate<Schema::CompletedBackups::Kind>(static_cast<ui8>(kind))); } else { db.Table<Schema::MigratedCompletedBackups>() .Key(pathId.OwnerId, pathId.LocalPathId, txId, - info.CompletionDateTime) - .Update(NIceDb::TUpdate<Schema::MigratedCompletedBackups::TotalShardCount>(info.TotalShardCount), - NIceDb::TUpdate<Schema::MigratedCompletedBackups::SuccessShardCount>(info.SuccessShardCount), - NIceDb::TUpdate<Schema::MigratedCompletedBackups::StartTime>(info.StartDateTime), - NIceDb::TUpdate<Schema::MigratedCompletedBackups::DataTotalSize>(txState.DataTotalSize), - NIceDb::TUpdate<Schema::MigratedCompletedBackups::Kind>(static_cast<ui8>(kind))); + info.CompletionDateTime) + .Update(NIceDb::TUpdate<Schema::MigratedCompletedBackups::TotalShardCount>(info.TotalShardCount), + NIceDb::TUpdate<Schema::MigratedCompletedBackups::SuccessShardCount>(info.SuccessShardCount), + NIceDb::TUpdate<Schema::MigratedCompletedBackups::StartTime>(info.StartDateTime), + NIceDb::TUpdate<Schema::MigratedCompletedBackups::DataTotalSize>(txState.DataTotalSize), + NIceDb::TUpdate<Schema::MigratedCompletedBackups::Kind>(static_cast<ui8>(kind))); } } void TSchemeShard::PersistCompletedBackup(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& backupInfo) { - PersistCompletedBackupRestore(db, txId, txState, backupInfo, TTableInfo::TBackupRestoreResult::EKind::Backup); -} - + PersistCompletedBackupRestore(db, txId, txState, backupInfo, TTableInfo::TBackupRestoreResult::EKind::Backup); +} + void TSchemeShard::PersistCompletedRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& restoreInfo) { - PersistCompletedBackupRestore(db, txId, txState, restoreInfo, TTableInfo::TBackupRestoreResult::EKind::Restore); -} - + PersistCompletedBackupRestore(db, txId, txState, restoreInfo, TTableInfo::TBackupRestoreResult::EKind::Restore); +} + void TSchemeShard::PersistBackupDone(NIceDb::TNiceDb& db, TPathId pathId) { if (IsLocalId(pathId)) { db.Table<Schema::BackupSettings>().Key(pathId.LocalPathId).Delete(); @@ -3004,52 +3004,52 @@ void TSchemeShard::PersistSequenceAlterRemove(NIceDb::TNiceDb& db, TPathId pathI db.Table<Schema::SequencesAlters>().Key(pathId.LocalPathId).Delete(); } -void TSchemeShard::PersistReplication(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo) { - Y_VERIFY(IsLocalId(pathId)); - - TString serializedDescription; - Y_VERIFY(replicationInfo.Description.SerializeToString(&serializedDescription)); - - db.Table<Schema::Replications>().Key(pathId.LocalPathId).Update( - NIceDb::TUpdate<Schema::Replications::AlterVersion>(replicationInfo.AlterVersion), - NIceDb::TUpdate<Schema::Replications::Description>(serializedDescription)); -} - -void TSchemeShard::PersistReplicationRemove(NIceDb::TNiceDb& db, TPathId pathId) { - Y_VERIFY(IsLocalId(pathId)); - - if (!Replications.contains(pathId)) { - return; - } - - auto replicationInfo = Replications.at(pathId); - if (replicationInfo->AlterData) { - replicationInfo->AlterData = nullptr; - PersistReplicationAlterRemove(db, pathId); - } - - Replications.erase(pathId); - DecrementPathDbRefCount(pathId); - db.Table<Schema::Replications>().Key(pathId.LocalPathId).Delete(); -} - -void TSchemeShard::PersistReplicationAlter(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo) { - Y_VERIFY(IsLocalId(pathId)); - - TString serializedDescription; - Y_VERIFY(replicationInfo.Description.SerializeToString(&serializedDescription)); - - db.Table<Schema::ReplicationsAlterData>().Key(pathId.LocalPathId).Update( - NIceDb::TUpdate<Schema::ReplicationsAlterData::AlterVersion>(replicationInfo.AlterVersion), - NIceDb::TUpdate<Schema::ReplicationsAlterData::Description>(serializedDescription)); -} - -void TSchemeShard::PersistReplicationAlterRemove(NIceDb::TNiceDb& db, TPathId pathId) { - Y_VERIFY(IsLocalId(pathId)); - - db.Table<Schema::ReplicationsAlterData>().Key(pathId.LocalPathId).Delete(); -} - +void TSchemeShard::PersistReplication(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo) { + Y_VERIFY(IsLocalId(pathId)); + + TString serializedDescription; + Y_VERIFY(replicationInfo.Description.SerializeToString(&serializedDescription)); + + db.Table<Schema::Replications>().Key(pathId.LocalPathId).Update( + NIceDb::TUpdate<Schema::Replications::AlterVersion>(replicationInfo.AlterVersion), + NIceDb::TUpdate<Schema::Replications::Description>(serializedDescription)); +} + +void TSchemeShard::PersistReplicationRemove(NIceDb::TNiceDb& db, TPathId pathId) { + Y_VERIFY(IsLocalId(pathId)); + + if (!Replications.contains(pathId)) { + return; + } + + auto replicationInfo = Replications.at(pathId); + if (replicationInfo->AlterData) { + replicationInfo->AlterData = nullptr; + PersistReplicationAlterRemove(db, pathId); + } + + Replications.erase(pathId); + DecrementPathDbRefCount(pathId); + db.Table<Schema::Replications>().Key(pathId.LocalPathId).Delete(); +} + +void TSchemeShard::PersistReplicationAlter(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo) { + Y_VERIFY(IsLocalId(pathId)); + + TString serializedDescription; + Y_VERIFY(replicationInfo.Description.SerializeToString(&serializedDescription)); + + db.Table<Schema::ReplicationsAlterData>().Key(pathId.LocalPathId).Update( + NIceDb::TUpdate<Schema::ReplicationsAlterData::AlterVersion>(replicationInfo.AlterVersion), + NIceDb::TUpdate<Schema::ReplicationsAlterData::Description>(serializedDescription)); +} + +void TSchemeShard::PersistReplicationAlterRemove(NIceDb::TNiceDb& db, TPathId pathId) { + Y_VERIFY(IsLocalId(pathId)); + + db.Table<Schema::ReplicationsAlterData>().Key(pathId.LocalPathId).Delete(); +} + void TSchemeShard::PersistKesusInfo(NIceDb::TNiceDb& db, TPathId pathId, const TKesusInfo::TPtr kesus) { TString config; @@ -3137,36 +3137,36 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const } const TTableInfo::TPtr tableInfo = Tables.at(pathId); - auto clearHistory = [&](const TMap<TTxId, TTableInfo::TBackupRestoreResult>& history) { - for (auto& bItem: history) { - TTxId txId = bItem.first; - const auto& result = bItem.second; - - for (auto& sItem: result.ShardStatuses) { - auto shard = sItem.first; - if (IsLocalId(shard)) { - db.Table<Schema::ShardBackupStatus>().Key(txId, shard.GetLocalId()).Delete(); - } - db.Table<Schema::MigratedShardBackupStatus>().Key(txId, shard.GetOwnerId(), shard.GetLocalId()).Delete(); - db.Table<Schema::TxShardStatus>().Key(txId, shard.GetOwnerId(), shard.GetLocalId()).Delete(); + auto clearHistory = [&](const TMap<TTxId, TTableInfo::TBackupRestoreResult>& history) { + for (auto& bItem: history) { + TTxId txId = bItem.first; + const auto& result = bItem.second; + + for (auto& sItem: result.ShardStatuses) { + auto shard = sItem.first; + if (IsLocalId(shard)) { + db.Table<Schema::ShardBackupStatus>().Key(txId, shard.GetLocalId()).Delete(); + } + db.Table<Schema::MigratedShardBackupStatus>().Key(txId, shard.GetOwnerId(), shard.GetLocalId()).Delete(); + db.Table<Schema::TxShardStatus>().Key(txId, shard.GetOwnerId(), shard.GetLocalId()).Delete(); } - if (IsLocalId(pathId)) { - db.Table<Schema::CompletedBackups>().Key(pathId.LocalPathId, txId, result.CompletionDateTime).Delete(); - } - db.Table<Schema::MigratedCompletedBackups>().Key(pathId.OwnerId, pathId.LocalPathId, txId, result.CompletionDateTime).Delete(); + if (IsLocalId(pathId)) { + db.Table<Schema::CompletedBackups>().Key(pathId.LocalPathId, txId, result.CompletionDateTime).Delete(); + } + db.Table<Schema::MigratedCompletedBackups>().Key(pathId.OwnerId, pathId.LocalPathId, txId, result.CompletionDateTime).Delete(); } - }; + }; + + clearHistory(tableInfo->BackupHistory); + clearHistory(tableInfo->RestoreHistory); - clearHistory(tableInfo->BackupHistory); - clearHistory(tableInfo->RestoreHistory); - if (IsLocalId(pathId)) { db.Table<Schema::BackupSettings>().Key(pathId.LocalPathId).Delete(); } db.Table<Schema::MigratedBackupSettings>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); - db.Table<Schema::RestoreTasks>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); + db.Table<Schema::RestoreTasks>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); if (tableInfo->AlterData) { auto& alterData = tableInfo->AlterData; @@ -3190,32 +3190,32 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const db.Table<Schema::TablePartitions>().Key(pathId.LocalPathId, pNo).Delete(); } db.Table<Schema::MigratedTablePartitions>().Key(pathId.OwnerId, pathId.LocalPathId, pNo).Delete(); - db.Table<Schema::TablePartitionStats>().Key(pathId.OwnerId, pathId.LocalPathId, pNo).Delete(); - - const auto& shardInfo = tableInfo->GetPartitions().at(pNo); - if (const auto& lag = shardInfo.LastCondEraseLag) { - TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); - } + db.Table<Schema::TablePartitionStats>().Key(pathId.OwnerId, pathId.LocalPathId, pNo).Delete(); + + const auto& shardInfo = tableInfo->GetPartitions().at(pNo); + if (const auto& lag = shardInfo.LastCondEraseLag) { + TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].DecrementFor(lag->Seconds()); + } } - for (const auto& [_, childPathId]: path->GetChildren()) { - Y_VERIFY(PathsById.contains(childPathId)); - auto childPath = PathsById.at(childPathId); - - if (childPath->IsTableIndex()) { - PersistRemoveTableIndex(db, childPathId); - } + for (const auto& [_, childPathId]: path->GetChildren()) { + Y_VERIFY(PathsById.contains(childPathId)); + auto childPath = PathsById.at(childPathId); + + if (childPath->IsTableIndex()) { + PersistRemoveTableIndex(db, childPathId); + } } if (pathId.OwnerId == TabletID()) { db.Table<Schema::Tables>().Key(pathId.LocalPathId).Delete(); } db.Table<Schema::MigratedTables>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); - - if (tableInfo->IsTTLEnabled()) { - TTLEnabledTables.erase(pathId); + + if (tableInfo->IsTTLEnabled()) { + TTLEnabledTables.erase(pathId); TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Sub(1); - } + } if (TablesWithSnaphots.contains(pathId)) { const TTxId snapshotId = TablesWithSnaphots.at(pathId); @@ -3230,18 +3230,18 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const } if (!tableInfo->IsBackup && !tableInfo->IsShardsStatsDetached()) { - auto subDomainId = ResolveDomainId(pathId); - auto subDomainInfo = ResolveDomainInfo(pathId); + auto subDomainId = ResolveDomainId(pathId); + auto subDomainInfo = ResolveDomainInfo(pathId); subDomainInfo->AggrDiskSpaceUsage(this, TTableInfo::TPartitionStats(), tableInfo->GetStats().Aggregated); if (subDomainInfo->CheckDiskSpaceQuotas(this)) { - PersistSubDomainState(db, subDomainId, *subDomainInfo); - // Publish is done in a separate transaction, so we may call this directly - TDeque<TPathId> toPublish; - toPublish.push_back(subDomainId); - PublishToSchemeBoard(TTxId(), std::move(toPublish), ctx); - } - } - + PersistSubDomainState(db, subDomainId, *subDomainInfo); + // Publish is done in a separate transaction, so we may call this directly + TDeque<TPathId> toPublish; + toPublish.push_back(subDomainId); + PublishToSchemeBoard(TTxId(), std::move(toPublish), ctx); + } + } + for (const auto& p: tableInfo->GetPartitions()) { CompactionQueue->Remove(TShardCompactionInfo(p.ShardIdx)); } @@ -3249,7 +3249,7 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const Tables.erase(pathId); DecrementPathDbRefCount(pathId, "remove table"); - if (AppData()->FeatureFlags.GetEnableSystemViews()) { + if (AppData()->FeatureFlags.GetEnableSystemViews()) { auto ev = MakeHolder<NSysView::TEvSysView::TEvRemoveTable>(GetDomainKey(pathId), pathId); Send(SysPartitionStatsCollector, ev.Release()); } @@ -3323,8 +3323,8 @@ void TSchemeShard::PersistPublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathI .Key(txId, pathId.OwnerId, pathId.LocalPathId, version) .Update(); } -} - +} + void TSchemeShard::PersistRemovePublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathId pathId, ui64 version) { DecrementPathDbRefCount(pathId, "remove publishing"); @@ -3336,9 +3336,9 @@ void TSchemeShard::PersistRemovePublishingPath(NIceDb::TNiceDb& db, TTxId txId, db.Table<Schema::MigratedPublishingPaths>() .Key(txId, pathId.OwnerId, pathId.LocalPathId, version) - .Delete(); -} - + .Delete(); +} + TTabletId TSchemeShard::GetGlobalHive(const TActorContext& ctx) const { auto domainsInfo = AppData(ctx)->DomainsInfo; @@ -3436,12 +3436,12 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co const auto pathEl = path.Base(); const auto pathId = pathEl->PathId; - - if (pathEl->Dropped()) { + + if (pathEl->Dropped()) { result.SetGeneralVersion(Max<ui64>()); return result; - } - + } + ui64 generalVersion = 0; if (pathEl->IsCreateFinished()) { @@ -3573,13 +3573,13 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co generalVersion += result.GetSequenceVersion(); break; } - case NKikimrSchemeOp::EPathType::EPathTypeReplication: { - auto it = Replications.find(pathId); - Y_VERIFY(it != Replications.end()); - result.SetReplicationVersion(it->second->AlterVersion); - generalVersion += result.GetReplicationVersion(); - break; - } + case NKikimrSchemeOp::EPathType::EPathTypeReplication: { + auto it = Replications.find(pathId); + Y_VERIFY(it != Replications.end()); + result.SetReplicationVersion(it->second->AlterVersion); + generalVersion += result.GetReplicationVersion(); + break; + } case NKikimrSchemeOp::EPathType::EPathTypeInvalid: { Y_UNREACHABLE(); } @@ -3589,7 +3589,7 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co result.SetChildrenVersion(pathEl->DirAlterVersion); //not only childrens but also acl children's version increases it generalVersion += result.GetChildrenVersion(); - + result.SetUserAttrsVersion(pathEl->UserAttrs->AlterVersion); generalVersion += result.GetUserAttrsVersion(); @@ -3627,7 +3627,7 @@ TActorId TSchemeShard::TPipeClientFactory::CreateClient(const TActorContext& ctx TSchemeShard::TSchemeShard(const TActorId &tablet, TTabletStorageInfo *info) : TActor(&TThis::StateInit) , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory) - , AllowConditionalEraseOperations(1, 0, 1) + , AllowConditionalEraseOperations(1, 0, 1) , AllowServerlessStorageBilling(0, 0, 1) , SplitSettings() , IsReadOnlyMode(false) @@ -3641,7 +3641,7 @@ TSchemeShard::TSchemeShard(const TActorId &tablet, TTabletStorageInfo *info) , CompactionStarter(this) , ShardDeleter(info->TabletID) , AllowDataColumnForIndexTable(0, 0, 1) - , EnableAsyncIndexes(0, 0, 1) + , EnableAsyncIndexes(0, 0, 1) , EnableSchemeTransactionsAtSchemeShard(0, 0, 1) { TabletCountersPtr.Reset(new TProtobufTabletCounters< @@ -3697,8 +3697,8 @@ bool TSchemeShard::IsShemeShardConfigured() const { } void TSchemeShard::Die(const TActorContext &ctx) { - ctx.Send(SchemeBoardPopulator, new TEvents::TEvPoisonPill()); - ctx.Send(TxAllocatorClient, new TEvents::TEvPoisonPill()); + ctx.Send(SchemeBoardPopulator, new TEvents::TEvPoisonPill()); + ctx.Send(TxAllocatorClient, new TEvents::TEvPoisonPill()); ctx.Send(SysPartitionStatsCollector, new TEvents::TEvPoisonPill()); ShardDeleter.Shutdown(ctx); @@ -3721,17 +3721,17 @@ void TSchemeShard::OnTabletDead(TEvTablet::TEvTabletDead::TPtr &ev, const TActor Die(ctx); } -static TVector<ui64> CollectTxAllocators(const TAppData *appData) { - TVector<ui64> allocators; - for (auto it: appData->DomainsInfo->Domains) { - auto &domain = it.second; - for (auto tabletId: domain->TxAllocators) { - allocators.push_back(tabletId); - } - } - return allocators; -} - +static TVector<ui64> CollectTxAllocators(const TAppData *appData) { + TVector<ui64> allocators; + for (auto it: appData->DomainsInfo->Domains) { + auto &domain = it.second; + for (auto tabletId: domain->TxAllocators) { + allocators.push_back(tabletId); + } + } + return allocators; +} + void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { const TTabletId selfTabletId = SelfTabletId(); @@ -3742,8 +3742,8 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { auto appData = AppData(ctx); Y_VERIFY(appData); - EnableBackgroundCompaction = appData->FeatureFlags.GetEnableBackgroundCompaction(); - EnableBackgroundCompactionServerless = appData->FeatureFlags.GetEnableBackgroundCompactionServerless(); + EnableBackgroundCompaction = appData->FeatureFlags.GetEnableBackgroundCompaction(); + EnableBackgroundCompactionServerless = appData->FeatureFlags.GetEnableBackgroundCompactionServerless(); if (!CompactionQueue) ConfigureCompactionQueue(appData->CompactionConfig.GetBackgroundCompactionConfig(), ctx); @@ -3753,26 +3753,26 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { ChannelProfiles = appData->ChannelProfiles; } - appData->Icb->RegisterSharedControl(AllowConditionalEraseOperations, "SchemeShard_AllowConditionalEraseOperations"); - - AllowDataColumnForIndexTable = appData->FeatureFlags.GetEnableDataColumnForIndexTable(); + appData->Icb->RegisterSharedControl(AllowConditionalEraseOperations, "SchemeShard_AllowConditionalEraseOperations"); + + AllowDataColumnForIndexTable = appData->FeatureFlags.GetEnableDataColumnForIndexTable(); appData->Icb->RegisterSharedControl(AllowDataColumnForIndexTable, "SchemeShard_AllowDataColumnForIndexTable"); - EnableAsyncIndexes = appData->FeatureFlags.GetEnableAsyncIndexes(); - appData->Icb->RegisterSharedControl(EnableAsyncIndexes, "SchemeShard_EnableAsyncIndexes"); - + EnableAsyncIndexes = appData->FeatureFlags.GetEnableAsyncIndexes(); + appData->Icb->RegisterSharedControl(EnableAsyncIndexes, "SchemeShard_EnableAsyncIndexes"); + EnableSchemeTransactionsAtSchemeShard = appData->FeatureFlags.GetEnableSchemeTransactionsAtSchemeShard(); appData->Icb->RegisterSharedControl(EnableSchemeTransactionsAtSchemeShard, "SchemeShard_EnableSchemeTransactionsAtSchemeShard"); - for (const auto& sid : appData->MeteringConfig.GetSystemBackupSIDs()) { - SystemBackupSIDs.insert(sid); - } - + for (const auto& sid : appData->MeteringConfig.GetSystemBackupSIDs()) { + SystemBackupSIDs.insert(sid); + } + AllowServerlessStorageBilling = appData->FeatureFlags.GetAllowServerlessStorageBillingForSchemeShard(); appData->Icb->RegisterSharedControl(AllowServerlessStorageBilling, "SchemeShard_AllowServerlessStorageBilling"); TxAllocatorClient = RegisterWithSameMailbox(CreateTxAllocatorClient(CollectTxAllocators(appData))); - + SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release()); SplitSettings.Register(appData->Icb); @@ -3905,14 +3905,14 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(NSequenceShard::TEvSequenceShard::TEvRestoreSequenceResult, Handle); HFuncTraced(NSequenceShard::TEvSequenceShard::TEvRedirectSequenceResult, Handle); - // replication - HFuncTraced(NReplication::TEvController::TEvCreateReplicationResult, Handle); - HFuncTraced(NReplication::TEvController::TEvDropReplicationResult, Handle); - - // conditional erase - HFuncTraced(TEvPrivate::TEvRunConditionalErase, Handle); - HFuncTraced(TEvDataShard::TEvConditionalEraseRowsResponse, Handle); - + // replication + HFuncTraced(NReplication::TEvController::TEvCreateReplicationResult, Handle); + HFuncTraced(NReplication::TEvController::TEvDropReplicationResult, Handle); + + // conditional erase + HFuncTraced(TEvPrivate::TEvRunConditionalErase, Handle); + HFuncTraced(TEvDataShard::TEvConditionalEraseRowsResponse, Handle); + HFuncTraced(TEvPrivate::TEvServerlessStorageBilling, Handle); HFuncTraced(NSysView::TEvSysView::TEvGetPartitionStats, Handle); @@ -3931,7 +3931,7 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvSchemeShard::TEvSyncTenantSchemeShard, Handle); HFuncTraced(TEvSchemeShard::TEvUpdateTenantSchemeShard, Handle); - HFuncTraced(TSchemeBoardEvents::TEvUpdateAck, Handle); + HFuncTraced(TSchemeBoardEvents::TEvUpdateAck, Handle); HFuncTraced(TEvBlockStore::TEvUpdateVolumeConfigResponse, Handle); HFuncTraced(TEvFileStore::TEvUpdateConfigResponse, Handle); @@ -3944,24 +3944,24 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvTabletPipe::TEvClientDestroyed, Handle); HFuncTraced(TEvTabletPipe::TEvServerConnected, Handle); HFuncTraced(TEvTabletPipe::TEvServerDisconnected, Handle); - - // namespace NExport { - HFuncTraced(TEvExport::TEvCreateExportRequest, Handle); - HFuncTraced(TEvExport::TEvGetExportRequest, Handle); - HFuncTraced(TEvExport::TEvCancelExportRequest, Handle); - HFuncTraced(TEvExport::TEvForgetExportRequest, Handle); - HFuncTraced(TEvExport::TEvListExportsRequest, Handle); + + // namespace NExport { + HFuncTraced(TEvExport::TEvCreateExportRequest, Handle); + HFuncTraced(TEvExport::TEvGetExportRequest, Handle); + HFuncTraced(TEvExport::TEvCancelExportRequest, Handle); + HFuncTraced(TEvExport::TEvForgetExportRequest, Handle); + HFuncTraced(TEvExport::TEvListExportsRequest, Handle); // } // NExport - - // namespace NImport { - HFuncTraced(TEvImport::TEvCreateImportRequest, Handle); - HFuncTraced(TEvImport::TEvGetImportRequest, Handle); - HFuncTraced(TEvImport::TEvCancelImportRequest, Handle); - HFuncTraced(TEvImport::TEvForgetImportRequest, Handle); - HFuncTraced(TEvImport::TEvListImportsRequest, Handle); - HFuncTraced(TEvPrivate::TEvImportSchemeReady, Handle); - // } // NImport - + + // namespace NImport { + HFuncTraced(TEvImport::TEvCreateImportRequest, Handle); + HFuncTraced(TEvImport::TEvGetImportRequest, Handle); + HFuncTraced(TEvImport::TEvCancelImportRequest, Handle); + HFuncTraced(TEvImport::TEvForgetImportRequest, Handle); + HFuncTraced(TEvImport::TEvListImportsRequest, Handle); + HFuncTraced(TEvPrivate::TEvImportSchemeReady, Handle); + // } // NImport + //namespace NIndexBuilder { HFuncTraced(TEvIndexBuilder::TEvCreateRequest, Handle); HFuncTraced(TEvIndexBuilder::TEvGetRequest, Handle); @@ -3970,17 +3970,17 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvIndexBuilder::TEvListRequest, Handle); HFuncTraced(TEvDataShard::TEvBuildIndexProgressResponse, Handle); HFuncTraced(TEvPrivate::TEvIndexBuildingMakeABill, Handle); - // } // NIndexBuilder + // } // NIndexBuilder - // namespace NLongRunningCommon { - HFuncTraced(TEvTxAllocatorClient::TEvAllocateResult, Handle); + // namespace NLongRunningCommon { + HFuncTraced(TEvTxAllocatorClient::TEvAllocateResult, Handle); HFuncTraced(TEvSchemeShard::TEvModifySchemeTransactionResult, Handle); - HFuncTraced(TEvIndexBuilder::TEvCreateResponse, Handle); + HFuncTraced(TEvIndexBuilder::TEvCreateResponse, Handle); HFuncTraced(TEvSchemeShard::TEvNotifyTxCompletionRegistered, Handle); HFuncTraced(TEvSchemeShard::TEvNotifyTxCompletionResult, Handle); HFuncTraced(TEvSchemeShard::TEvCancelTxResult, Handle); - HFuncTraced(TEvIndexBuilder::TEvCancelResponse, Handle); - // } // NLongRunningCommon + HFuncTraced(TEvIndexBuilder::TEvCancelResponse, Handle); + // } // NLongRunningCommon //console configs HFuncTraced(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle); @@ -4269,15 +4269,15 @@ void TSchemeShard::UncountNode(TPathElement::TPtr node) { case TPathElement::EPathType::EPathTypeColumnTable: // TODO break; - case TPathElement::EPathType::EPathTypeCdcStream: + case TPathElement::EPathType::EPathTypeCdcStream: TabletCounters->Simple()[COUNTER_CDC_STREAMS_COUNT].Sub(1); - break; + break; case TPathElement::EPathType::EPathTypeSequence: TabletCounters->Simple()[COUNTER_SEQUENCE_COUNT].Sub(1); break; - case TPathElement::EPathType::EPathTypeReplication: - TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Sub(1); - break; + case TPathElement::EPathType::EPathTypeReplication: + TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Sub(1); + break; case TPathElement::EPathType::EPathTypeInvalid: Y_FAIL("imposible path type"); } @@ -4381,7 +4381,7 @@ TString TSchemeShard::FillBackupTxBody(TPathId pathId, const NKikimrSchemeOp::TB NKikimrTxDataShard::TFlatSchemeTransaction tx; FillSeqNo(tx, seqNo); auto backup = tx.MutableBackup(); - backup->CopyFrom(task); + backup->CopyFrom(task); backup->SetTableId(pathId.LocalPathId); backup->SetShardNum(shardNum); @@ -4639,13 +4639,13 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvUpdateTenantSchemeShard::TPtr& ev, } void TSchemeShard::Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle TEvUpdateAck" - << ", at schemeshard: " << TabletID() - << ", msg: " << record.ShortDebugString() - << ", cookie: " << ev->Cookie); - + const auto& record = ev->Get()->Record; + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle TEvUpdateAck" + << ", at schemeshard: " << TabletID() + << ", msg: " << record.ShortDebugString() + << ", cookie: " << ev->Cookie); + const auto pathId = TPathId(ev->Get()->Record.GetPathOwnerId(), ev->Get()->Record.GetLocalPathId()); if (DelayedInitTenantReply && DelayedInitTenantDestination && pathId == RootPathId()) { ctx.Send(DelayedInitTenantDestination, DelayedInitTenantReply.Release()); @@ -4658,17 +4658,17 @@ void TSchemeShard::Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev, const TAct return; } - if (!Operations.contains(txId) && !Publications.contains(txId)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got TEvUpdateAck" - << " for unknown txId " << txId - << ", at schemeshard: " << TabletID()); - return; - } - - Execute(CreateTxAckPublishToSchemeBoard(ev), ctx); -} - + if (!Operations.contains(txId) && !Publications.contains(txId)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Got TEvUpdateAck" + << " for unknown txId " << txId + << ", at schemeshard: " << TabletID()); + return; + } + + Execute(CreateTxAckPublishToSchemeBoard(ev), ctx); +} + void TSchemeShard::Handle(TEvTxProcessing::TEvPlanStep::TPtr &ev, const TActorContext &ctx) { Execute(CreateTxOperationPlanStep(ev), ctx); } @@ -4787,18 +4787,18 @@ void TSchemeShard::Handle(TEvPersQueue::TEvDropTabletReply::TPtr &ev, const TAct return; } - auto tabletId = TTabletId(ev->Get()->Record.GetTabletId()); - TSubTxId partId = Operations.at(txId)->FindRelatedPartByTabletId(tabletId, ctx); - if (partId == InvalidSubTxId) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got TEvPersQueue::TEvDropTabletReply but partId is unknown" - << ", for txId: " << txId - << ", tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - - Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); + auto tabletId = TTabletId(ev->Get()->Record.GetTabletId()); + TSubTxId partId = Operations.at(txId)->FindRelatedPartByTabletId(tabletId, ctx); + if (partId == InvalidSubTxId) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Got TEvPersQueue::TEvDropTabletReply but partId is unknown" + << ", for txId: " << txId + << ", tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + + Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); } void TSchemeShard::Handle(TEvPersQueue::TEvUpdateConfigResponse::TPtr &ev, const TActorContext &ctx) { @@ -4815,7 +4815,7 @@ void TSchemeShard::Handle(TEvPersQueue::TEvUpdateConfigResponse::TPtr &ev, const TSubTxId partId = Operations.at(txId)->FindRelatedPartByTabletId(tabletId, ctx); if (partId == InvalidSubTxId) { LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got TEvUpdateConfigResponse but partId is unknown" + "Got TEvUpdateConfigResponse but partId is unknown" << ", for txId: " << txId << ", tabletId: " << tabletId << ", at schemeshard: " << TabletID()); @@ -4949,28 +4949,28 @@ void TSchemeShard::Handle(NSequenceShard::TEvSequenceShard::TEvRedirectSequenceR Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); } -void TSchemeShard::Handle(NReplication::TEvController::TEvCreateReplicationResult::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle TEvCreateReplicationResult" - << ", at schemeshard: " << TabletID() - << ", message: " << ev->Get()->Record.ShortDebugString()); - - const auto txId = TTxId(ev->Get()->Record.GetOperationId().GetTxId()); - const auto partId = TSubTxId(ev->Get()->Record.GetOperationId().GetPartId()); - Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); -} - -void TSchemeShard::Handle(NReplication::TEvController::TEvDropReplicationResult::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle TEvDropReplicationResult" - << ", at schemeshard: " << TabletID() - << ", message: " << ev->Get()->Record.ShortDebugString()); - - const auto txId = TTxId(ev->Get()->Record.GetOperationId().GetTxId()); - const auto partId = TSubTxId(ev->Get()->Record.GetOperationId().GetPartId()); - Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); -} - +void TSchemeShard::Handle(NReplication::TEvController::TEvCreateReplicationResult::TPtr &ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle TEvCreateReplicationResult" + << ", at schemeshard: " << TabletID() + << ", message: " << ev->Get()->Record.ShortDebugString()); + + const auto txId = TTxId(ev->Get()->Record.GetOperationId().GetTxId()); + const auto partId = TSubTxId(ev->Get()->Record.GetOperationId().GetPartId()); + Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); +} + +void TSchemeShard::Handle(NReplication::TEvController::TEvDropReplicationResult::TPtr &ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle TEvDropReplicationResult" + << ", at schemeshard: " << TabletID() + << ", message: " << ev->Get()->Record.ShortDebugString()); + + const auto txId = TTxId(ev->Get()->Record.GetOperationId().GetTxId()); + const auto partId = TSubTxId(ev->Get()->Record.GetOperationId().GetPartId()); + Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); +} + void TSchemeShard::Handle(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Handle TEvProposeTransactionResult" @@ -5016,13 +5016,13 @@ void TSchemeShard::Handle(TEvSubDomain::TEvConfigureStatus::TPtr &ev, const TAct if (opId.GetSubTxId() == InvalidSubTxId) { LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got TEvSubDomain::TEvConfigureStatus but partId in unknown" + "Got TEvSubDomain::TEvConfigureStatus but partId in unknown" << ", for txId: " << opId.GetTxId() - << ", tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - + << ", tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + Execute(CreateTxOperationReply(opId, ev), ctx); } @@ -5038,16 +5038,16 @@ void TSchemeShard::Handle(TEvBlockStore::TEvUpdateVolumeConfigResponse::TPtr& ev auto tabletId = TTabletId(ev->Get()->Record.GetOrigin()); auto partId = Operations.at(txId)->FindRelatedPartByTabletId(tabletId, ctx); - if (partId == InvalidSubTxId) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got TEvUpdateVolumeConfigResponse but partId in unknown" - << ", for txId: " << txId - << ", tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - - Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); + if (partId == InvalidSubTxId) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Got TEvUpdateVolumeConfigResponse but partId in unknown" + << ", for txId: " << txId + << ", tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + + Execute(CreateTxOperationReply(TOperationId(txId, partId), ev), ctx); } void TSchemeShard::Handle(TEvFileStore::TEvUpdateConfigResponse::TPtr& ev, const TActorContext& ctx) { @@ -5176,13 +5176,13 @@ void TSchemeShard::Handle(NKesus::TEvKesus::TEvSetConfigResult::TPtr& ev, const if (opId.GetSubTxId() == InvalidSubTxId) { LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Got NKesus::TEvKesus::TEvSetConfigResult but partId in unknown" + "Got NKesus::TEvKesus::TEvSetConfigResult but partId in unknown" << ", for txId: " << opId.GetTxId() - << ", tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - + << ", tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + Execute(CreateTxOperationReply(opId, ev), ctx); } @@ -5336,16 +5336,16 @@ void TSchemeShard::Handle(TEvDataShard::TEvMigrateSchemeShardResponse::TPtr &ev, } void TSchemeShard::ScheduleConditionalEraseRun(const TActorContext& ctx) { - ctx.Schedule(TDuration::Minutes(1), new TEvPrivate::TEvRunConditionalErase()); -} - + ctx.Schedule(TDuration::Minutes(1), new TEvPrivate::TEvRunConditionalErase()); +} + void TSchemeShard::Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Handle: TEvRunConditionalErase" - << ", at schemeshard: " << TabletID()); - - Execute(CreateTxRunConditionalErase(ev), ctx); -} - + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Handle: TEvRunConditionalErase" + << ", at schemeshard: " << TabletID()); + + Execute(CreateTxRunConditionalErase(ev), ctx); +} + void TSchemeShard::ScheduleServerlessStorageBilling(const TActorContext &ctx) { ctx.Send(SelfId(), new TEvPrivate::TEvServerlessStorageBilling()); } @@ -5355,27 +5355,27 @@ void TSchemeShard::Handle(TEvPrivate::TEvServerlessStorageBilling::TPtr &, const } void TSchemeShard::Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - const TTabletId tabletId(record.GetTabletID()); - const TShardIdx shardIdx = GetShardIdx(tabletId); - - if (!ShardInfos.contains(shardIdx)) { - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" - << ": tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - - if (record.GetStatus() == NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ACCEPTED) { - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase accepted" - << ": tabletId: " << tabletId - << ", at schemeshard: " << TabletID()); - return; - } - - Execute(CreateTxScheduleConditionalErase(ev), ctx); -} - + const auto& record = ev->Get()->Record; + const TTabletId tabletId(record.GetTabletID()); + const TShardIdx shardIdx = GetShardIdx(tabletId); + + if (!ShardInfos.contains(shardIdx)) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Unable to resolve shard info" + << ": tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + + if (record.GetStatus() == NKikimrTxDataShard::TEvConditionalEraseRowsResponse::ACCEPTED) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Conditional erase accepted" + << ": tabletId: " << tabletId + << ", at schemeshard: " << TabletID()); + return; + } + + Execute(CreateTxScheduleConditionalErase(ev), ctx); +} + void TSchemeShard::Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, const TActorContext& ctx) { LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Handle: TEvAllocateResult" @@ -5389,9 +5389,9 @@ void TSchemeShard::Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, con } return; } else if (Exports.contains(id)) { - return Execute(CreateTxProgressExport(ev), ctx); - } else if (Imports.contains(id)) { - return Execute(CreateTxProgressImport(ev), ctx); + return Execute(CreateTxProgressExport(ev), ctx); + } else if (Imports.contains(id)) { + return Execute(CreateTxProgressImport(ev), ctx); } else if (IndexBuilds.contains(TIndexBuildId(id))) { return Execute(CreateTxReply(ev), ctx); } @@ -5413,9 +5413,9 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr const auto txId = TTxId(ev->Get()->Record.GetTxId()); if (TxIdToExport.contains(txId)) { - return Execute(CreateTxProgressExport(ev), ctx); - } else if (TxIdToImport.contains(txId)) { - return Execute(CreateTxProgressImport(ev), ctx); + return Execute(CreateTxProgressExport(ev), ctx); + } else if (TxIdToImport.contains(txId)) { + return Execute(CreateTxProgressImport(ev), ctx); } else if (TxIdToIndexBuilds.contains(txId)) { return Execute(CreateTxReply(ev), ctx); } @@ -5427,29 +5427,29 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr } void TSchemeShard::Handle(TEvIndexBuilder::TEvCreateResponse::TPtr& ev, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle: TEvIndexBuilder::TEvCreateResponse" - << ": txId# " << ev->Get()->Record.GetTxId() - << ", status# " << ev->Get()->Record.GetStatus()); - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Message:\n" << ev->Get()->Record.ShortDebugString()); - - const auto txId = TTxId(ev->Get()->Record.GetTxId()); - - if (TxIdToImport.contains(txId)) { - return Execute(CreateTxProgressImport(ev), ctx); - } - - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "no able to determine destination for message TEvIndexBuilder::TEvCreateResponse: " - << " txId: " << txId - << ", at schemeshard: " << TabletID()); -} - + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle: TEvIndexBuilder::TEvCreateResponse" + << ": txId# " << ev->Get()->Record.GetTxId() + << ", status# " << ev->Get()->Record.GetStatus()); + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Message:\n" << ev->Get()->Record.ShortDebugString()); + + const auto txId = TTxId(ev->Get()->Record.GetTxId()); + + if (TxIdToImport.contains(txId)) { + return Execute(CreateTxProgressImport(ev), ctx); + } + + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "no able to determine destination for message TEvIndexBuilder::TEvCreateResponse: " + << " txId: " << txId + << ", at schemeshard: " << TabletID()); +} + void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr&, const TActorContext&) { - // just ignore -} - + // just ignore +} + void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) { LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Handle: TEvNotifyTxCompletionResult" @@ -5460,9 +5460,9 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const auto txId = TTxId(ev->Get()->Record.GetTxId()); if (TxIdToExport.contains(txId)) { - return Execute(CreateTxProgressExport(ev), ctx); - } else if (TxIdToImport.contains(txId)) { - return Execute(CreateTxProgressImport(ev), ctx); + return Execute(CreateTxProgressExport(ev), ctx); + } else if (TxIdToImport.contains(txId)) { + return Execute(CreateTxProgressImport(ev), ctx); } else if (TxIdToIndexBuilds.contains(txId)) { return Execute(CreateTxReply(ev), ctx); } @@ -5474,45 +5474,45 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, } void TSchemeShard::Handle(TEvSchemeShard::TEvCancelTxResult::TPtr& ev, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle: TEvCancelTxResult" - << ": Cookie: " << ev->Cookie - << ", at schemeshard: " << TabletID()); - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Message:\n" << ev->Get()->Record.ShortDebugString()); - - const ui64 id = ev->Cookie; - if (Exports.contains(id)) { - return Execute(CreateTxCancelExportAck(ev), ctx); - } else if (Imports.contains(id)) { - return Execute(CreateTxCancelImportAck(ev), ctx); - } - - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "no able to determine destination for message TEvCancelTxResult" - << ": Cookie: " << id - << ", at schemeshard: " << TabletID()); -} - + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle: TEvCancelTxResult" + << ": Cookie: " << ev->Cookie + << ", at schemeshard: " << TabletID()); + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Message:\n" << ev->Get()->Record.ShortDebugString()); + + const ui64 id = ev->Cookie; + if (Exports.contains(id)) { + return Execute(CreateTxCancelExportAck(ev), ctx); + } else if (Imports.contains(id)) { + return Execute(CreateTxCancelImportAck(ev), ctx); + } + + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "no able to determine destination for message TEvCancelTxResult" + << ": Cookie: " << id + << ", at schemeshard: " << TabletID()); +} + void TSchemeShard::Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Handle: TEvIndexBuilder::TEvCancelResponse" - << ": Cookie: " << ev->Cookie - << ", at schemeshard: " << TabletID()); - LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Message:\n" << ev->Get()->Record.ShortDebugString()); - - const ui64 id = ev->Cookie; - if (Imports.contains(id)) { - return Execute(CreateTxCancelImportAck(ev), ctx); - } - - LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "no able to determine destination for message TEvIndexBuilder::TEvCancelResponse" - << ": Cookie: " << id - << ", at schemeshard: " << TabletID()); -} - + LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Handle: TEvIndexBuilder::TEvCancelResponse" + << ": Cookie: " << ev->Cookie + << ", at schemeshard: " << TabletID()); + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Message:\n" << ev->Get()->Record.ShortDebugString()); + + const ui64 id = ev->Cookie; + if (Imports.contains(id)) { + return Execute(CreateTxCancelImportAck(ev), ctx); + } + + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "no able to determine destination for message TEvIndexBuilder::TEvCancelResponse" + << ": Cookie: " << id + << ", at schemeshard: " << TabletID()); +} + void TSchemeShard::FillSeqNo(NKikimrTxDataShard::TFlatSchemeTransaction& tx, TMessageSeqNo seqNo) { tx.MutableSeqNo()->SetGeneration(seqNo.Generation); tx.MutableSeqNo()->SetRound(seqNo.Round); @@ -5540,7 +5540,7 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T proto->SetName(path->Name); proto->SetId_Deprecated(pathId.LocalPathId); - PathIdFromPathId(pathId, proto->MutablePathId()); + PathIdFromPathId(pathId, proto->MutablePathId()); for (const auto& col : alterData->Columns) { const TTableInfo::TColumn& colInfo = col.second; @@ -5627,7 +5627,7 @@ void TSchemeShard::ApplyPartitionConfigStoragePatch( void TSchemeShard::FillTableDescriptionForShardIdx( TPathId tableId, TShardIdx shardIdx, NKikimrSchemeOp::TTableDescription* tableDescr, TString rangeBegin, TString rangeEnd, - bool rangeBeginInclusive, bool rangeEndInclusive, bool newTable) + bool rangeBeginInclusive, bool rangeEndInclusive, bool newTable) { Y_VERIFY_S(Tables.contains(tableId), "Unknown table id " << tableId); const TTableInfo::TPtr tinfo = Tables.at(tableId); @@ -5656,40 +5656,40 @@ void TSchemeShard::FillTableDescriptionForShardIdx( *tableDescr->MutablePartitionConfig(), *patch); } - + if (tinfo->IsBackup) { tableDescr->SetIsBackup(true); } - // Fill indexes & cdc streams (if any) - for (const auto& child : pinfo->GetChildren()) { - const auto& childName = child.first; - const auto& childPathId = child.second; - - Y_VERIFY(PathsById.contains(childPathId)); - auto childPath = PathsById.at(childPathId); - - if (childPath->Dropped() || childPath->PlannedToDrop()) { - continue; - } - - switch (childPath->PathType) { + // Fill indexes & cdc streams (if any) + for (const auto& child : pinfo->GetChildren()) { + const auto& childName = child.first; + const auto& childPathId = child.second; + + Y_VERIFY(PathsById.contains(childPathId)); + auto childPath = PathsById.at(childPathId); + + if (childPath->Dropped() || childPath->PlannedToDrop()) { + continue; + } + + switch (childPath->PathType) { case NKikimrSchemeOp::EPathTypeTableIndex: { - Y_VERIFY(Indexes.contains(childPathId)); - auto info = Indexes.at(childPathId); - DescribeTableIndex(childPathId, childName, newTable ? info->AlterData : info, *tableDescr->MutableTableIndexes()->Add()); - break; - } - + Y_VERIFY(Indexes.contains(childPathId)); + auto info = Indexes.at(childPathId); + DescribeTableIndex(childPathId, childName, newTable ? info->AlterData : info, *tableDescr->MutableTableIndexes()->Add()); + break; + } + case NKikimrSchemeOp::EPathTypeCdcStream: { - Y_VERIFY_S(CdcStreams.contains(childPathId), "Cdc stream not found" - << ": pathId# " << childPathId - << ", name# " << childName); - auto info = CdcStreams.at(childPathId); - DescribeCdcStream(childPathId, childName, info, *tableDescr->MutableCdcStreams()->Add()); - break; - } - + Y_VERIFY_S(CdcStreams.contains(childPathId), "Cdc stream not found" + << ": pathId# " << childPathId + << ", name# " << childName); + auto info = CdcStreams.at(childPathId); + DescribeCdcStream(childPathId, childName, info, *tableDescr->MutableCdcStreams()->Add()); + break; + } + case NKikimrSchemeOp::EPathTypeSequence: { Y_VERIFY_S(Sequences.contains(childPathId), "Sequence not found" << ": path#d# " << childPathId @@ -5699,14 +5699,14 @@ void TSchemeShard::FillTableDescriptionForShardIdx( break; } - default: - Y_FAIL_S("Unexpected table's child" - << ": tableId# " << tableId - << ", childId# " << childPathId - << ", childName# " << childName - << ", childType# " << static_cast<ui32>(childPath->PathType)); - } - } + default: + Y_FAIL_S("Unexpected table's child" + << ": tableId# " << tableId + << ", childId# " << childPathId + << ", childName# " << childName + << ", childType# " << static_cast<ui32>(childPath->PathType)); + } + } } // Fills CreateTable transaction that is sent to datashards @@ -5728,7 +5728,7 @@ void TSchemeShard::FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 tableDescr, std::move(rangeBegin), std::move(rangeEnd), - true /* rangeBeginInclusive */, false /* rangeEndInclusive */, true /* newTable */); + true /* rangeBeginInclusive */, false /* rangeEndInclusive */, true /* newTable */); FillTableSchemaVersion(schemaVersion, tableDescr); } @@ -5765,7 +5765,7 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key } void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning) { - if (AppData()->FeatureFlags.GetEnableSystemViews()) { + if (AppData()->FeatureFlags.GetEnableSystemViews()) { TVector<std::pair<ui64, ui64>> shardIndices; shardIndices.reserve(newPartitioning.size()); for (auto& info : newPartitioning) { @@ -5797,23 +5797,23 @@ void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, T } void TSchemeShard::FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx) { - Y_VERIFY(PathsById.contains(tableId)); - - auto parent = TPath::Init(tableId, this).Parent(); - Y_VERIFY(parent.IsResolved()); - - if (!parent.Base()->IsTableIndex()) { - return; - } - - Y_VERIFY(Indexes.contains(parent.Base()->PathId)); - auto index = Indexes.at(parent.Base()->PathId); - - if (index->Type == TTableIndexInfo::EType::EIndexTypeGlobalAsync) { - tx.MutableAsyncIndexInfo(); - } -} - + Y_VERIFY(PathsById.contains(tableId)); + + auto parent = TPath::Init(tableId, this).Parent(); + Y_VERIFY(parent.IsResolved()); + + if (!parent.Base()->IsTableIndex()) { + return; + } + + Y_VERIFY(Indexes.contains(parent.Base()->PathId)); + auto index = Indexes.at(parent.Base()->PathId); + + if (index->Type == TTableIndexInfo::EType::EIndexTypeGlobalAsync) { + tx.MutableAsyncIndexInfo(); + } +} + bool TSchemeShard::ReadSysValue(NIceDb::TNiceDb &db, ui64 sysTag, TString &value, TString defValue) { auto sysParamsRowset = db.Table<Schema::SysParams>().Key(sysTag).Select<Schema::SysParams::Value>(); if (!sysParamsRowset.IsReady()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 1512b469102..e1a1d08b9a6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1,10 +1,10 @@ #pragma once #include "schemeshard.h" -#include "schemeshard_export.h" -#include "schemeshard_import.h" +#include "schemeshard_export.h" +#include "schemeshard_import.h" #include "schemeshard_build_index.h" -#include "schemeshard_private.h" +#include "schemeshard_private.h" #include "schemeshard_types.h" #include "schemeshard_path_element.h" #include "schemeshard_path.h" @@ -40,7 +40,7 @@ #include <ydb/core/tx/message_seqno.h> #include <ydb/core/tx/scheme_board/events.h> #include <ydb/core/tx/tx_allocator_client/actor_client.h> -#include <ydb/core/tx/replication/controller/public_events.h> +#include <ydb/core/tx/replication/controller/public_events.h> #include <ydb/core/tx/sequenceshard/public/events.h> #include <ydb/core/tx/tx_processing.h> #include <ydb/core/util/pb.h> @@ -105,13 +105,13 @@ public: static constexpr ui32 MaxPQGroupPartitionsCount = 20*1000; static constexpr ui32 MaxPQWriteSpeedPerPartition = 50*1024*1024; static constexpr ui32 MaxPQLifetimeSeconds = 31 * 86400; - static constexpr ui32 PublishChunkSize = 1000; + static constexpr ui32 PublishChunkSize = 1000; static const TSchemeLimits DefaultLimits; TIntrusivePtr<TChannelProfiles> ChannelProfiles; - TControlWrapper AllowConditionalEraseOperations; + TControlWrapper AllowConditionalEraseOperations; TControlWrapper AllowServerlessStorageBilling; TSplitSettings SplitSettings; @@ -138,7 +138,7 @@ public: TEffectiveACL ParentDomainCachedEffectiveACL; TString ParentDomainOwner; - THashSet<TString> SystemBackupSIDs; + THashSet<TString> SystemBackupSIDs; TInstant ServerlessStorageLastBillTime; TParentDomainLink ParentDomainLink; @@ -150,13 +150,13 @@ public: TLocalPathId NextLocalPathId = 0; THashMap<TPathId, TTableInfo::TPtr> Tables; - THashMap<TPathId, TTableInfo::TPtr> TTLEnabledTables; + THashMap<TPathId, TTableInfo::TPtr> TTLEnabledTables; - THashMap<TPathId, TTableIndexInfo::TPtr> Indexes; - THashMap<TPathId, TCdcStreamInfo::TPtr> CdcStreams; + THashMap<TPathId, TTableIndexInfo::TPtr> Indexes; + THashMap<TPathId, TCdcStreamInfo::TPtr> CdcStreams; THashMap<TPathId, TSequenceInfo::TPtr> Sequences; - THashMap<TPathId, TReplicationInfo::TPtr> Replications; - + THashMap<TPathId, TReplicationInfo::TPtr> Replications; + THashMap<TPathId, TTxId> TablesWithSnaphots; THashMap<TTxId, TSet<TPathId>> SnapshotTables; THashMap<TTxId, TStepId> SnapshotsStepIds; @@ -177,7 +177,7 @@ public: THashMap<TPathId, TVector<TTabletId>> RevertedMigrations; THashMap<TTxId, TOperation::TPtr> Operations; - THashMap<TTxId, TPublicationInfo> Publications; + THashMap<TTxId, TPublicationInfo> Publications; THashMap<TOperationId, TTxState> TxInFlight; ui64 NextLocalShardIdx = 0; @@ -187,11 +187,11 @@ public: THashMap<TShardIdx, TVector<TActorId>> ShardDeletionSubscribers; // for tests TActorId SchemeBoardPopulator; - + static constexpr ui32 InitiateCachedTxIdsCount = 100; TDeque<TTxId> CachedTxIds; TActorId TxAllocatorClient; - + TAutoPtr<NTabletPipe::IClientCache> PipeClientCache; TPipeTracker PipeTracker; @@ -230,11 +230,11 @@ public: return pId == RootPathId(); } - bool IsServerlessDomain(const TPath& domain) const { - const auto& resourcesDomainId = domain.DomainInfo()->GetResourcesDomainId(); - return !IsDomainSchemeShard && resourcesDomainId && resourcesDomainId != ParentDomainId; - } - + bool IsServerlessDomain(const TPath& domain) const { + const auto& resourcesDomainId = domain.DomainInfo()->GetResourcesDomainId(); + return !IsDomainSchemeShard && resourcesDomainId && resourcesDomainId != ParentDomainId; + } + TPathId MakeLocalId(const TLocalPathId& localPathId) const { return TPathId(TabletID(), localPathId); } @@ -413,7 +413,7 @@ public: void DoShardsDeletion(const THashSet<TShardIdx>& shardIdx, const TActorContext& ctx); void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning); - auto BuildStatsForCollector(TPathId tableId, TShardIdx shardIdx, TTabletId datashardId, + auto BuildStatsForCollector(TPathId tableId, TShardIdx shardIdx, TTabletId datashardId, TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats); bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, TString& value, TString defValue = TString()); @@ -442,21 +442,21 @@ public: void PersistTableIndex(NIceDb::TNiceDb& db, const TPathId& pathId); void PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId& pathId); - // cdc stream - void PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId); - void PersistCdcStreamAlterData(NIceDb::TNiceDb& db, const TPathId& pathId); - void PersistRemoveCdcStream(NIceDb::TNiceDb& db, const TPathId& tableId); - + // cdc stream + void PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId); + void PersistCdcStreamAlterData(NIceDb::TNiceDb& db, const TPathId& pathId); + void PersistRemoveCdcStream(NIceDb::TNiceDb& db, const TPathId& tableId); + static void PersistTxMinStep(NIceDb::TNiceDb& db, const TOperationId opId, TStepId minStep); void PersistRemoveTx(NIceDb::TNiceDb& db, const TOperationId opId, const TTxState& txState); void PersistTable(NIceDb::TNiceDb &db, const TPathId pathId); void PersistChannelsBinding(NIceDb::TNiceDb& db, const TShardIdx shardId, const TChannelsBindings& bindedChannels); void PersistTablePartitioning(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo); void DeleteTablePartitioning(NIceDb::TNiceDb& db, const TPathId tableId, const TTableInfo::TPtr tableInfo); - void PersistTablePartitionCondErase(NIceDb::TNiceDb& db, const TPathId& pathId, ui64 id, const TTableInfo::TPtr tableInfo); - void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, ui64 partitionId, const TTableInfo::TPartitionStats& stats); - void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TShardIdx& shardIdx, const TTableInfo::TPtr tableInfo); - void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TTableInfo::TPtr tableInfo); + void PersistTablePartitionCondErase(NIceDb::TNiceDb& db, const TPathId& pathId, ui64 id, const TTableInfo::TPtr tableInfo); + void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, ui64 partitionId, const TTableInfo::TPartitionStats& stats); + void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TShardIdx& shardIdx, const TTableInfo::TPtr tableInfo); + void PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId& tableId, const TTableInfo::TPtr tableInfo); void PersistTableCreated(NIceDb::TNiceDb& db, const TPathId tableId); void PersistTableAlterVersion(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo); void PersistTableAltered(NIceDb::TNiceDb &db, const TPathId pathId, const TTableInfo::TPtr tableInfo); @@ -465,7 +465,7 @@ public: void PersistRemovePersQueueGroup(NIceDb::TNiceDb &db, TPathId pathId); void PersistAddPersQueueGroupAlter(NIceDb::TNiceDb &db, TPathId pathId, const TPersQueueGroupInfo::TPtr); void PersistRemovePersQueueGroupAlter(NIceDb::TNiceDb &db, TPathId pathId); - void PersistPersQueue(NIceDb::TNiceDb &db, TPathId pathId, TShardIdx shardIdx, const TPQShardInfo::TPersQueueInfo& pqInfo); + void PersistPersQueue(NIceDb::TNiceDb &db, TPathId pathId, TShardIdx shardIdx, const TPQShardInfo::TPersQueueInfo& pqInfo); void PersistRemovePersQueue(NIceDb::TNiceDb &db, TPathId pathId, ui32 pqId); void PersistRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId, const TRtmrVolumeInfo::TPtr rtmrVol); void PersistRemoveRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId); @@ -489,7 +489,7 @@ public: void PersistTxState(NIceDb::TNiceDb& db, const TOperationId opId); void ChangeTxState(NIceDb::TNiceDb& db, const TOperationId opId, TTxState::ETxState newState); - void PersistCancelTx(NIceDb::TNiceDb& db, const TOperationId opId, const TTxState& txState); + void PersistCancelTx(NIceDb::TNiceDb& db, const TOperationId opId, const TTxState& txState); void PersistTxPlanStep(NIceDb::TNiceDb& db, TOperationId opId, TStepId step); @@ -501,12 +501,12 @@ public: void PersistShardsToDelete(NIceDb::TNiceDb& db, const THashSet<TShardIdx>& shardsIdxs); void PersistShardDeleted(NIceDb::TNiceDb& db, TShardIdx shardIdx, const TChannelsBindings& bindedChannels); void PersistUnknownShardDeleted(NIceDb::TNiceDb& db, TShardIdx shardIdx); - void PersistTxShardStatus(NIceDb::TNiceDb& db, TOperationId opId, TShardIdx shardIdx, const TTxState::TShardStatus& status); + void PersistTxShardStatus(NIceDb::TNiceDb& db, TOperationId opId, TShardIdx shardIdx, const TTxState::TShardStatus& status); void PersistBackupSettings(NIceDb::TNiceDb& db, TPathId pathId, const NKikimrSchemeOp::TBackupTask& settings); void PersistBackupDone(NIceDb::TNiceDb& db, TPathId pathId); - void PersistCompletedBackupRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& info, TTableInfo::TBackupRestoreResult::EKind kind); - void PersistCompletedBackup(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& backupInfo); - void PersistCompletedRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& restoreInfo); + void PersistCompletedBackupRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& info, TTableInfo::TBackupRestoreResult::EKind kind); + void PersistCompletedBackup(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& backupInfo); + void PersistCompletedRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& restoreInfo); void PersistSchemeLimit(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain); void PersistStoragePools(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain); void PersistSubDomain(NIceDb::TNiceDb& db, const TPathId& pathId, const TSubDomainInfo& subDomain); @@ -561,17 +561,17 @@ public: void PersistSequenceAlter(NIceDb::TNiceDb& db, TPathId pathId, const TSequenceInfo& sequenceInfo); void PersistSequenceAlterRemove(NIceDb::TNiceDb& db, TPathId pathId); - // Replication - void PersistReplication(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo); - void PersistReplicationRemove(NIceDb::TNiceDb& db, TPathId pathId); - void PersistReplicationAlter(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo); - void PersistReplicationAlterRemove(NIceDb::TNiceDb& db, TPathId pathId); - + // Replication + void PersistReplication(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo); + void PersistReplicationRemove(NIceDb::TNiceDb& db, TPathId pathId); + void PersistReplicationAlter(NIceDb::TNiceDb& db, TPathId pathId, const TReplicationInfo& replicationInfo); + void PersistReplicationAlterRemove(NIceDb::TNiceDb& db, TPathId pathId); + void PersistAddTableShardPartitionConfig(NIceDb::TNiceDb& db, TShardIdx shardIdx, const NKikimrSchemeOp::TPartitionConfig& config); void PersistPublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathId pathId, ui64 version); void PersistRemovePublishingPath(NIceDb::TNiceDb& db, TTxId txId, TPathId pathId, ui64 version); - + void PersistInitState(NIceDb::TNiceDb& db); @@ -607,7 +607,7 @@ public: TDeque<TPathId>&& blockStoreVolumesToClean = {} ); - struct TTxInitPopulator; + struct TTxInitPopulator; NTabletFlatExecutor::ITransaction* CreateTxInitPopulator(TSideEffects::TPublications&& publications); struct TTxInitSchema; @@ -636,7 +636,7 @@ public: void ScheduleCleanDroppedSubDomains(); void Handle(TEvPrivate::TEvCleanDroppedSubDomains::TPtr& ev, const TActorContext& ctx); - struct TTxFixBadPaths; + struct TTxFixBadPaths; NTabletFlatExecutor::ITransaction* CreateTxFixBadPaths(); struct TTxPublishTenantAsReadOnly; @@ -660,26 +660,26 @@ public: struct TTxShardStateChanged; NTabletFlatExecutor::ITransaction* CreateTxShardStateChanged(TEvDataShard::TEvStateChanged::TPtr& ev); - struct TTxRunConditionalErase; - NTabletFlatExecutor::ITransaction* CreateTxRunConditionalErase(TEvPrivate::TEvRunConditionalErase::TPtr& ev); - - struct TTxScheduleConditionalErase; - NTabletFlatExecutor::ITransaction* CreateTxScheduleConditionalErase(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev); - + struct TTxRunConditionalErase; + NTabletFlatExecutor::ITransaction* CreateTxRunConditionalErase(TEvPrivate::TEvRunConditionalErase::TPtr& ev); + + struct TTxScheduleConditionalErase; + NTabletFlatExecutor::ITransaction* CreateTxScheduleConditionalErase(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev); + struct TTxSyncTenant; NTabletFlatExecutor::ITransaction* CreateTxSyncTenant(TPathId tabletId); struct TTxUpdateTenant; NTabletFlatExecutor::ITransaction* CreateTxUpdateTenant(TEvSchemeShard::TEvUpdateTenantSchemeShard::TPtr& ev); - struct TTxPublishToSchemeBoard; - NTabletFlatExecutor::ITransaction* CreateTxPublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths); - struct TTxAckPublishToSchemeBoard; - NTabletFlatExecutor::ITransaction* CreateTxAckPublishToSchemeBoard(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev); - + struct TTxPublishToSchemeBoard; + NTabletFlatExecutor::ITransaction* CreateTxPublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths); + struct TTxAckPublishToSchemeBoard; + NTabletFlatExecutor::ITransaction* CreateTxAckPublishToSchemeBoard(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev); + struct TTxOperationPropose; NTabletFlatExecutor::ITransaction* CreateTxOperationPropose(TEvSchemeShard::TEvModifySchemeTransaction::TPtr& ev); - struct TTxOperationProposeCancelTx; + struct TTxOperationProposeCancelTx; NTabletFlatExecutor::ITransaction* CreateTxOperationPropose(TEvSchemeShard::TEvCancelTx::TPtr& ev); struct TTxOperationProgress; @@ -707,8 +707,8 @@ public: SCHEMESHARD_INCOMING_EVENTS(DeclareCreateTxOperationReply) #undef DeclareCreateTxOperationReply - void PublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths, const TActorContext& ctx); - void PublishToSchemeBoard(TTxId txId, TDeque<TPathId>&& paths, const TActorContext& ctx); + void PublishToSchemeBoard(THashMap<TTxId, TDeque<TPathId>>&& paths, const TActorContext& ctx); + void PublishToSchemeBoard(TTxId txId, TDeque<TPathId>&& paths, const TActorContext& ctx); void ApplyPartitionConfigStoragePatch( NKikimrSchemeOp::TPartitionConfig& config, @@ -716,8 +716,8 @@ public: void FillTableDescriptionForShardIdx( TPathId tableId, TShardIdx shardIdx, NKikimrSchemeOp::TTableDescription* tableDescr, TString rangeBegin, TString rangeEnd, - bool rangeBeginInclusive, bool rangeEndInclusive, - bool newTable = false); + bool rangeBeginInclusive, bool rangeEndInclusive, + bool newTable = false); void FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 schemaVersion, NKikimrSchemeOp::TTableDescription* tableDescr); static bool FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeId firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr); @@ -731,8 +731,8 @@ public: static void FillSeqNo(NKikimrTxDataShard::TFlatSchemeTransaction &tx, TMessageSeqNo seqNo); static void FillSeqNo(NKikimrTxColumnShard::TSchemaTxBody &tx, TMessageSeqNo seqNo); - void FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx); - + void FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx); + void DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme::TTypeRegistry* typeRegistry, bool fillConfig, bool fillBoundaries, NKikimrSchemeOp::TTableDescription* entry) const; void DescribeTableIndex(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TIndexDescription& entry); @@ -741,8 +741,8 @@ public: void DescribeCdcStream(const TPathId& pathId, const TString& name, TCdcStreamInfo::TPtr info, NKikimrSchemeOp::TCdcStreamDescription& desc); void DescribeSequence(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TSequenceDescription& desc); void DescribeSequence(const TPathId& pathId, const TString& name, TSequenceInfo::TPtr info, NKikimrSchemeOp::TSequenceDescription& desc); - void DescribeReplication(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TReplicationDescription& desc); - void DescribeReplication(const TPathId& pathId, const TString& name, TReplicationInfo::TPtr info, NKikimrSchemeOp::TReplicationDescription& desc); + void DescribeReplication(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TReplicationDescription& desc); + void DescribeReplication(const TPathId& pathId, const TString& name, TReplicationInfo::TPtr info, NKikimrSchemeOp::TReplicationDescription& desc); static void FillTableBoundaries(const TTableInfo::TPtr tableInfo, google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries); void Handle(TEvSchemeShard::TEvInitRootShard::TPtr &ev, const TActorContext &ctx); @@ -775,8 +775,8 @@ public: void Handle(NSequenceShard::TEvSequenceShard::TEvFreezeSequenceResult::TPtr &ev, const TActorContext &ctx); void Handle(NSequenceShard::TEvSequenceShard::TEvRestoreSequenceResult::TPtr &ev, const TActorContext &ctx); void Handle(NSequenceShard::TEvSequenceShard::TEvRedirectSequenceResult::TPtr &ev, const TActorContext &ctx); - void Handle(NReplication::TEvController::TEvCreateReplicationResult::TPtr &ev, const TActorContext &ctx); - void Handle(NReplication::TEvController::TEvDropReplicationResult::TPtr &ev, const TActorContext &ctx); + void Handle(NReplication::TEvController::TEvCreateReplicationResult::TPtr &ev, const TActorContext &ctx); + void Handle(NReplication::TEvController::TEvDropReplicationResult::TPtr &ev, const TActorContext &ctx); void Handle(TEvDataShard::TEvProposeTransactionResult::TPtr &ev, const TActorContext &ctx); void Handle(TEvDataShard::TEvSchemaChanged::TPtr &ev, const TActorContext &ctx); void Handle(TEvDataShard::TEvStateChanged::TPtr &ev, const TActorContext &ctx); @@ -798,7 +798,7 @@ public: void Handle(TEvSchemeShard::TEvSyncTenantSchemeShard::TPtr& ev, const TActorContext& ctx); void Handle(TEvSchemeShard::TEvUpdateTenantSchemeShard::TPtr& ev, const TActorContext& ctx); - void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev, const TActorContext& ctx); + void Handle(TSchemeBoardEvents::TEvUpdateAck::TPtr& ev, const TActorContext& ctx); void Handle(TEvTxProcessing::TEvPlanStep::TPtr &ev, const TActorContext &ctx); @@ -814,10 +814,10 @@ public: void Handle(TEvSchemeShard::TEvFindTabletSubDomainPathId::TPtr& ev, const TActorContext& ctx); - void ScheduleConditionalEraseRun(const TActorContext& ctx); - void Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx); - + void ScheduleConditionalEraseRun(const TActorContext& ctx); + void Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx); + void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx); void ScheduleServerlessStorageBilling(const TActorContext& ctx); @@ -832,122 +832,122 @@ public: TOperationId RouteIncomming(TTabletId tabletId, const TActorContext& ctx); - // namespace NLongRunningCommon { - struct TXxport { - class TTxBase; - template <typename TInfo, typename TEvRequest, typename TEvResponse> struct TTxGet; - template <typename TInfo, typename TEvRequest, typename TEvResponse, typename TDerived> struct TTxList; - }; - - void Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, const TActorContext& ctx); + // namespace NLongRunningCommon { + struct TXxport { + class TTxBase; + template <typename TInfo, typename TEvRequest, typename TEvResponse> struct TTxGet; + template <typename TInfo, typename TEvRequest, typename TEvResponse, typename TDerived> struct TTxList; + }; + + void Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvIndexBuilder::TEvCreateResponse::TPtr& ev, const TActorContext& ctx); + void Handle(TEvIndexBuilder::TEvCreateResponse::TPtr& ev, const TActorContext& ctx); void Handle(TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr& ev, const TActorContext& ctx); void Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvSchemeShard::TEvCancelTxResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev, const TActorContext& ctx); - // } // NLongRunningCommon - - // namespace NExport { - THashMap<ui64, TExportInfo::TPtr> Exports; - THashMap<TString, TExportInfo::TPtr> ExportsByUid; - THashMap<TTxId, std::pair<ui64, ui32>> TxIdToExport; - - void FromXxportInfo(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo); - - static void PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); - static void PersistRemoveExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); - static void PersistExportPathId(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); - static void PersistExportState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); - static void PersistExportItemState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo, ui32 targetIdx); - - struct TExport { - struct TTxCreate; - struct TTxGet; - struct TTxCancel; - struct TTxCancelAck; - struct TTxForget; - struct TTxList; - - struct TTxProgress; - }; - - NTabletFlatExecutor::ITransaction* CreateTxCreateExport(TEvExport::TEvCreateExportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxGetExport(TEvExport::TEvGetExportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxCancelExport(TEvExport::TEvCancelExportRequest::TPtr& ev); + void Handle(TEvIndexBuilder::TEvCancelResponse::TPtr& ev, const TActorContext& ctx); + // } // NLongRunningCommon + + // namespace NExport { + THashMap<ui64, TExportInfo::TPtr> Exports; + THashMap<TString, TExportInfo::TPtr> ExportsByUid; + THashMap<TTxId, std::pair<ui64, ui32>> TxIdToExport; + + void FromXxportInfo(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo); + + static void PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); + static void PersistRemoveExport(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); + static void PersistExportPathId(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); + static void PersistExportState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo); + static void PersistExportItemState(NIceDb::TNiceDb& db, const TExportInfo::TPtr exportInfo, ui32 targetIdx); + + struct TExport { + struct TTxCreate; + struct TTxGet; + struct TTxCancel; + struct TTxCancelAck; + struct TTxForget; + struct TTxList; + + struct TTxProgress; + }; + + NTabletFlatExecutor::ITransaction* CreateTxCreateExport(TEvExport::TEvCreateExportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxGetExport(TEvExport::TEvGetExportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxCancelExport(TEvExport::TEvCancelExportRequest::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxCancelExportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxForgetExport(TEvExport::TEvForgetExportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxListExports(TEvExport::TEvListExportsRequest::TPtr& ev); - - NTabletFlatExecutor::ITransaction* CreateTxProgressExport(ui64 id); - NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxForgetExport(TEvExport::TEvForgetExportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxListExports(TEvExport::TEvListExportsRequest::TPtr& ev); + + NTabletFlatExecutor::ITransaction* CreateTxProgressExport(ui64 id); + NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev); - - void Handle(TEvExport::TEvCreateExportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExport::TEvGetExportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExport::TEvCancelExportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExport::TEvForgetExportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TActorContext& ctx); - - void ResumeExports(const TVector<ui64>& exportIds, const TActorContext& ctx); - // } // NExport - - // namespace NImport { - THashMap<ui64, TImportInfo::TPtr> Imports; - THashMap<TString, TImportInfo::TPtr> ImportsByUid; - THashMap<TTxId, std::pair<ui64, ui32>> TxIdToImport; - - void FromXxportInfo(NKikimrImport::TImport& exprt, const TImportInfo::TPtr importInfo); - - static void PersistCreateImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); - static void PersistRemoveImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); - static void PersistImportState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); - static void PersistImportItemState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); - static void PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); - static void PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); - - struct TImport { - struct TTxCreate; - struct TTxGet; - struct TTxCancel; - struct TTxCancelAck; - struct TTxForget; - struct TTxList; - - struct TTxProgress; - }; - - NTabletFlatExecutor::ITransaction* CreateTxCreateImport(TEvImport::TEvCreateImportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxGetImport(TEvImport::TEvGetImportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxCancelImport(TEvImport::TEvCancelImportRequest::TPtr& ev); + + void Handle(TEvExport::TEvCreateExportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExport::TEvGetExportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExport::TEvCancelExportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExport::TEvForgetExportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TActorContext& ctx); + + void ResumeExports(const TVector<ui64>& exportIds, const TActorContext& ctx); + // } // NExport + + // namespace NImport { + THashMap<ui64, TImportInfo::TPtr> Imports; + THashMap<TString, TImportInfo::TPtr> ImportsByUid; + THashMap<TTxId, std::pair<ui64, ui32>> TxIdToImport; + + void FromXxportInfo(NKikimrImport::TImport& exprt, const TImportInfo::TPtr importInfo); + + static void PersistCreateImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); + static void PersistRemoveImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); + static void PersistImportState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo); + static void PersistImportItemState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); + static void PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); + static void PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx); + + struct TImport { + struct TTxCreate; + struct TTxGet; + struct TTxCancel; + struct TTxCancelAck; + struct TTxForget; + struct TTxList; + + struct TTxProgress; + }; + + NTabletFlatExecutor::ITransaction* CreateTxCreateImport(TEvImport::TEvCreateImportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxGetImport(TEvImport::TEvGetImportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxCancelImport(TEvImport::TEvCancelImportRequest::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxCancelImportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxCancelImportAck(TEvIndexBuilder::TEvCancelResponse::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxForgetImport(TEvImport::TEvForgetImportRequest::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxListImports(TEvImport::TEvListImportsRequest::TPtr& ev); - - NTabletFlatExecutor::ITransaction* CreateTxProgressImport(ui64 id); - NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvPrivate::TEvImportSchemeReady::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxCancelImportAck(TEvIndexBuilder::TEvCancelResponse::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxForgetImport(TEvImport::TEvForgetImportRequest::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxListImports(TEvImport::TEvListImportsRequest::TPtr& ev); + + NTabletFlatExecutor::ITransaction* CreateTxProgressImport(ui64 id); + NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvPrivate::TEvImportSchemeReady::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev); - NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvIndexBuilder::TEvCreateResponse::TPtr& ev); + NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvIndexBuilder::TEvCreateResponse::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressImport(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev); - - void Handle(TEvImport::TEvCreateImportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImport::TEvGetImportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx); - - void ResumeImports(const TVector<ui64>& ids, const TActorContext& ctx); - // } // NImport - + + void Handle(TEvImport::TEvCreateImportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImport::TEvGetImportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx); + + void ResumeImports(const TVector<ui64>& ids, const TActorContext& ctx); + // } // NImport + void FillTableSchemaVersion(ui64 schemaVersion, NKikimrSchemeOp::TTableDescription *tableDescr) const; - + // namespace NIndexBuilder { TControlWrapper AllowDataColumnForIndexTable; - TControlWrapper EnableAsyncIndexes; + TControlWrapper EnableAsyncIndexes; TControlWrapper EnableSchemeTransactionsAtSchemeShard; THashMap<TIndexBuildId, TIndexBuildInfo::TPtr> IndexBuilds; @@ -976,33 +976,33 @@ public: }; TDedicatedPipePool IndexBuildPipes; - void PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexIssue(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexCancelRequest(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexInitiateTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexInitiateTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexInitiateTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexLockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexLockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexLockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexApplyTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexApplyTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexApplyTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexUnlockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexUnlockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - void PersistBuildIndexUnlockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexUploadProgress(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx); - void PersistBuildIndexUploadInitiate(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx); + void PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexIssue(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexCancelRequest(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + + void PersistBuildIndexInitiateTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexInitiateTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexInitiateTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + + void PersistBuildIndexLockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexLockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexLockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + + void PersistBuildIndexApplyTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexApplyTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexApplyTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + + void PersistBuildIndexUnlockTxId(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexUnlockTxStatus(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + void PersistBuildIndexUnlockTxDone(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + + void PersistBuildIndexUploadProgress(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx); + void PersistBuildIndexUploadInitiate(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo, const TShardIdx& shardIdx); void PersistBuildIndexBilling(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - - void PersistBuildIndexForget(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); - + + void PersistBuildIndexForget(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr indexInfo); + struct TIndexBuilder { class TTxBase; diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 896fe33f499..c5e7d6c3d1e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -1,200 +1,200 @@ -#include "schemeshard_import.h" -#include "schemeshard_import_helpers.h" -#include "schemeshard_impl.h" - -#include <util/generic/xrange.h> - -namespace NKikimr { +#include "schemeshard_import.h" +#include "schemeshard_import_helpers.h" +#include "schemeshard_impl.h" + +#include <util/generic/xrange.h> + +namespace NKikimr { namespace NSchemeShard { - -namespace { - - void FillIssues(NKikimrImport::TImport& import, const TImportInfo::TPtr importInfo) { - if (importInfo->Issue) { - AddIssue(import, importInfo->Issue); - } - - for (const auto& item : importInfo->Items) { - if (item.Issue) { - AddIssue(import, item.Issue); - } - } - } - - TImportInfo::EState GetMinState(TImportInfo::TPtr importInfo) { - TImportInfo::EState state = TImportInfo::EState::Invalid; - - for (const auto& item : importInfo->Items) { - if (state == TImportInfo::EState::Invalid) { - state = item.State; - } - - state = Min(state, item.State); - } - - return state; - } - -} // anonymous - + +namespace { + + void FillIssues(NKikimrImport::TImport& import, const TImportInfo::TPtr importInfo) { + if (importInfo->Issue) { + AddIssue(import, importInfo->Issue); + } + + for (const auto& item : importInfo->Items) { + if (item.Issue) { + AddIssue(import, item.Issue); + } + } + } + + TImportInfo::EState GetMinState(TImportInfo::TPtr importInfo) { + TImportInfo::EState state = TImportInfo::EState::Invalid; + + for (const auto& item : importInfo->Items) { + if (state == TImportInfo::EState::Invalid) { + state = item.State; + } + + state = Min(state, item.State); + } + + return state; + } + +} // anonymous + void TSchemeShard::FromXxportInfo(NKikimrImport::TImport& import, const TImportInfo::TPtr importInfo) { - import.SetId(importInfo->Id); - import.SetStatus(Ydb::StatusIds::SUCCESS); - - switch (importInfo->State) { - case TImportInfo::EState::Waiting: - switch (GetMinState(importInfo)) { - case TImportInfo::EState::GetScheme: - case TImportInfo::EState::CreateTable: - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_PREPARING); - break; - case TImportInfo::EState::Transferring: - // TODO(ilnaz): fill items progress - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA); - break; - case TImportInfo::EState::BuildIndexes: - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES); - break; - case TImportInfo::EState::Done: - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_DONE); - break; - default: - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED); - break; - } - break; - - case TImportInfo::EState::Done: - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_DONE); - break; - - case TImportInfo::EState::Cancellation: - FillIssues(import, importInfo); - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_CANCELLATION); - break; - - case TImportInfo::EState::Cancelled: - import.SetStatus(Ydb::StatusIds::CANCELLED); - FillIssues(import, importInfo); - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_CANCELLED); - break; - - default: - import.SetStatus(Ydb::StatusIds::UNDETERMINED); - import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED); - break; - } - - switch (importInfo->Kind) { - case TImportInfo::EKind::S3: - import.MutableImportFromS3Settings()->CopyFrom(importInfo->Settings); - import.MutableImportFromS3Settings()->clear_access_key(); - import.MutableImportFromS3Settings()->clear_secret_key(); - break; - } -} - + import.SetId(importInfo->Id); + import.SetStatus(Ydb::StatusIds::SUCCESS); + + switch (importInfo->State) { + case TImportInfo::EState::Waiting: + switch (GetMinState(importInfo)) { + case TImportInfo::EState::GetScheme: + case TImportInfo::EState::CreateTable: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_PREPARING); + break; + case TImportInfo::EState::Transferring: + // TODO(ilnaz): fill items progress + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA); + break; + case TImportInfo::EState::BuildIndexes: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES); + break; + case TImportInfo::EState::Done: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_DONE); + break; + default: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED); + break; + } + break; + + case TImportInfo::EState::Done: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_DONE); + break; + + case TImportInfo::EState::Cancellation: + FillIssues(import, importInfo); + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_CANCELLATION); + break; + + case TImportInfo::EState::Cancelled: + import.SetStatus(Ydb::StatusIds::CANCELLED); + FillIssues(import, importInfo); + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_CANCELLED); + break; + + default: + import.SetStatus(Ydb::StatusIds::UNDETERMINED); + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED); + break; + } + + switch (importInfo->Kind) { + case TImportInfo::EKind::S3: + import.MutableImportFromS3Settings()->CopyFrom(importInfo->Settings); + import.MutableImportFromS3Settings()->clear_access_key(); + import.MutableImportFromS3Settings()->clear_secret_key(); + break; + } +} + void TSchemeShard::PersistCreateImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo) { - db.Table<Schema::Imports>().Key(importInfo->Id).Update( - NIceDb::TUpdate<Schema::Imports::Uid>(importInfo->Uid), - NIceDb::TUpdate<Schema::Imports::Kind>(static_cast<ui8>(importInfo->Kind)), - NIceDb::TUpdate<Schema::Imports::Settings>(importInfo->Settings.SerializeAsString()), - NIceDb::TUpdate<Schema::Imports::DomainPathOwnerId>(importInfo->DomainPathId.OwnerId), - NIceDb::TUpdate<Schema::Imports::DomainPathLocalId>(importInfo->DomainPathId.LocalPathId), - NIceDb::TUpdate<Schema::Imports::Items>(importInfo->Items.size()) - ); - - if (importInfo->UserSID) { - db.Table<Schema::Imports>().Key(importInfo->Id).Update( - NIceDb::TUpdate<Schema::Imports::UserSID>(*importInfo->UserSID) - ); - } - - for (ui32 itemIdx : xrange(importInfo->Items.size())) { - const auto& item = importInfo->Items.at(itemIdx); - - db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ImportItems::DstPathName>(item.DstPathName), - NIceDb::TUpdate<Schema::ImportItems::State>(static_cast<ui8>(item.State)) - ); - } -} - + db.Table<Schema::Imports>().Key(importInfo->Id).Update( + NIceDb::TUpdate<Schema::Imports::Uid>(importInfo->Uid), + NIceDb::TUpdate<Schema::Imports::Kind>(static_cast<ui8>(importInfo->Kind)), + NIceDb::TUpdate<Schema::Imports::Settings>(importInfo->Settings.SerializeAsString()), + NIceDb::TUpdate<Schema::Imports::DomainPathOwnerId>(importInfo->DomainPathId.OwnerId), + NIceDb::TUpdate<Schema::Imports::DomainPathLocalId>(importInfo->DomainPathId.LocalPathId), + NIceDb::TUpdate<Schema::Imports::Items>(importInfo->Items.size()) + ); + + if (importInfo->UserSID) { + db.Table<Schema::Imports>().Key(importInfo->Id).Update( + NIceDb::TUpdate<Schema::Imports::UserSID>(*importInfo->UserSID) + ); + } + + for (ui32 itemIdx : xrange(importInfo->Items.size())) { + const auto& item = importInfo->Items.at(itemIdx); + + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ImportItems::DstPathName>(item.DstPathName), + NIceDb::TUpdate<Schema::ImportItems::State>(static_cast<ui8>(item.State)) + ); + } +} + void TSchemeShard::PersistRemoveImport(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo) { - for (ui32 itemIdx : xrange(importInfo->Items.size())) { - db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Delete(); - } - - db.Table<Schema::Imports>().Key(importInfo->Id).Delete(); -} - + for (ui32 itemIdx : xrange(importInfo->Items.size())) { + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Delete(); + } + + db.Table<Schema::Imports>().Key(importInfo->Id).Delete(); +} + void TSchemeShard::PersistImportState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo) { - db.Table<Schema::Imports>().Key(importInfo->Id).Update( - NIceDb::TUpdate<Schema::Imports::State>(static_cast<ui8>(importInfo->State)), - NIceDb::TUpdate<Schema::Imports::Issue>(importInfo->Issue) - ); -} - + db.Table<Schema::Imports>().Key(importInfo->Id).Update( + NIceDb::TUpdate<Schema::Imports::State>(static_cast<ui8>(importInfo->State)), + NIceDb::TUpdate<Schema::Imports::Issue>(importInfo->Issue) + ); +} + void TSchemeShard::PersistImportItemState(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ImportItems::State>(static_cast<ui8>(item.State)), - NIceDb::TUpdate<Schema::ImportItems::WaitTxId>(item.WaitTxId), - NIceDb::TUpdate<Schema::ImportItems::NextIndexIdx>(item.NextIndexIdx), - NIceDb::TUpdate<Schema::ImportItems::Issue>(item.Issue) - ); -} - + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ImportItems::State>(static_cast<ui8>(item.State)), + NIceDb::TUpdate<Schema::ImportItems::WaitTxId>(item.WaitTxId), + NIceDb::TUpdate<Schema::ImportItems::NextIndexIdx>(item.NextIndexIdx), + NIceDb::TUpdate<Schema::ImportItems::Issue>(item.Issue) + ); +} + void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ImportItems::Scheme>(item.Scheme.SerializeAsString()) - ); -} - + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ImportItems::Scheme>(item.Scheme.SerializeAsString()) + ); +} + void TSchemeShard::PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate<Schema::ImportItems::DstPathOwnerId>(item.DstPathId.OwnerId), - NIceDb::TUpdate<Schema::ImportItems::DstPathLocalId>(item.DstPathId.LocalPathId) - ); -} - + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ImportItems::DstPathOwnerId>(item.DstPathId.OwnerId), + NIceDb::TUpdate<Schema::ImportItems::DstPathLocalId>(item.DstPathId.LocalPathId) + ); +} + void TSchemeShard::Handle(TEvImport::TEvCreateImportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxCreateImport(ev), ctx); -} - + Execute(CreateTxCreateImport(ev), ctx); +} + void TSchemeShard::Handle(TEvImport::TEvGetImportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxGetImport(ev), ctx); -} - + Execute(CreateTxGetImport(ev), ctx); +} + void TSchemeShard::Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxCancelImport(ev), ctx); -} - + Execute(CreateTxCancelImport(ev), ctx); +} + void TSchemeShard::Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxForgetImport(ev), ctx); -} - + Execute(CreateTxForgetImport(ev), ctx); +} + void TSchemeShard::Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxListImports(ev), ctx); -} - + Execute(CreateTxListImports(ev), ctx); +} + void TSchemeShard::Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxProgressImport(ev), ctx); -} - + Execute(CreateTxProgressImport(ev), ctx); +} + void TSchemeShard::ResumeImports(const TVector<ui64>& ids, const TActorContext& ctx) { - for (const ui64 id : ids) { - Execute(CreateTxProgressImport(id), ctx); - } -} - + for (const ui64 id : ids) { + Execute(CreateTxProgressImport(id), ctx); + } +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import.h b/ydb/core/tx/schemeshard/schemeshard_import.h index dc20e45b3da..41e45e57689 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.h +++ b/ydb/core/tx/schemeshard/schemeshard_import.h @@ -1,171 +1,171 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/protos/import.pb.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - -struct TEvImport { - enum EEv { - EvCreateImportRequest = EventSpaceBegin(TKikimrEvents::ES_IMPORT_SERVICE), - EvCreateImportResponse, - EvGetImportRequest, - EvGetImportResponse, - EvCancelImportRequest, - EvCancelImportResponse, - EvForgetImportRequest, - EvForgetImportResponse, - EvListImportsRequest, - EvListImportsResponse, - - EvEnd - }; - - static_assert( - EvEnd < EventSpaceEnd(TKikimrEvents::ES_IMPORT_SERVICE), - "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_IMPORT_SERVICE)" - ); - -#ifdef DECLARE_EVENT_CLASS -#error DECLARE_EVENT_CLASS macro redefinition -#else -#define DECLARE_EVENT_CLASS(NAME) struct T##NAME: public TEventPB<T##NAME, NKikimrImport::T##NAME, NAME> -#endif - - DECLARE_EVENT_CLASS(EvCreateImportRequest) { - TEvCreateImportRequest() = default; - - explicit TEvCreateImportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrImport::TCreateImportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - }; - - DECLARE_EVENT_CLASS(EvCreateImportResponse) { - TEvCreateImportResponse() = default; - - explicit TEvCreateImportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvGetImportRequest) { - TEvGetImportRequest() = default; - - explicit TEvGetImportRequest(const TString& dbName, const NKikimrImport::TGetImportRequest& request) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvGetImportRequest(const TString& dbName, const ui64 importId) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(importId); - } - }; - - DECLARE_EVENT_CLASS(EvGetImportResponse) { - }; - - DECLARE_EVENT_CLASS(EvCancelImportRequest) { - TEvCancelImportRequest() = default; - - explicit TEvCancelImportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrImport::TCancelImportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvCancelImportRequest( - const ui64 txId, - const TString& dbName, - const ui64 importId - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(importId); - } - }; - - DECLARE_EVENT_CLASS(EvCancelImportResponse) { - TEvCancelImportResponse() = default; - - explicit TEvCancelImportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvForgetImportRequest) { - TEvForgetImportRequest() = default; - - explicit TEvForgetImportRequest( - const ui64 txId, - const TString& dbName, - const NKikimrImport::TForgetImportRequest& request - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvForgetImportRequest( - const ui64 txId, - const TString& dbName, - const ui64 importId - ) { - Record.SetTxId(txId); - Record.SetDatabaseName(dbName); - Record.MutableRequest()->SetId(importId); - } - }; - - DECLARE_EVENT_CLASS(EvForgetImportResponse) { - TEvForgetImportResponse() = default; - - explicit TEvForgetImportResponse(const ui64 txId) { - Record.SetTxId(txId); - } - }; - - DECLARE_EVENT_CLASS(EvListImportsRequest) { - TEvListImportsRequest() = default; - - explicit TEvListImportsRequest(const TString& dbName, const NKikimrImport::TListImportsRequest& request) { - Record.SetDatabaseName(dbName); - Record.MutableRequest()->CopyFrom(request); - } - - explicit TEvListImportsRequest( - const TString& dbName, - const ui64 pageSize, - const TString& pageToken, - const TString& kind - ) { - Record.SetDatabaseName(dbName); - - auto& request = *Record.MutableRequest(); - request.SetPageSize(pageSize); - request.SetPageToken(pageToken); - request.SetKind(kind); - } - }; - - DECLARE_EVENT_CLASS(EvListImportsResponse) { - }; - -#undef DECLARE_EVENT_CLASS - -}; // TEvImport - + +struct TEvImport { + enum EEv { + EvCreateImportRequest = EventSpaceBegin(TKikimrEvents::ES_IMPORT_SERVICE), + EvCreateImportResponse, + EvGetImportRequest, + EvGetImportResponse, + EvCancelImportRequest, + EvCancelImportResponse, + EvForgetImportRequest, + EvForgetImportResponse, + EvListImportsRequest, + EvListImportsResponse, + + EvEnd + }; + + static_assert( + EvEnd < EventSpaceEnd(TKikimrEvents::ES_IMPORT_SERVICE), + "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_IMPORT_SERVICE)" + ); + +#ifdef DECLARE_EVENT_CLASS +#error DECLARE_EVENT_CLASS macro redefinition +#else +#define DECLARE_EVENT_CLASS(NAME) struct T##NAME: public TEventPB<T##NAME, NKikimrImport::T##NAME, NAME> +#endif + + DECLARE_EVENT_CLASS(EvCreateImportRequest) { + TEvCreateImportRequest() = default; + + explicit TEvCreateImportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrImport::TCreateImportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + }; + + DECLARE_EVENT_CLASS(EvCreateImportResponse) { + TEvCreateImportResponse() = default; + + explicit TEvCreateImportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvGetImportRequest) { + TEvGetImportRequest() = default; + + explicit TEvGetImportRequest(const TString& dbName, const NKikimrImport::TGetImportRequest& request) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvGetImportRequest(const TString& dbName, const ui64 importId) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(importId); + } + }; + + DECLARE_EVENT_CLASS(EvGetImportResponse) { + }; + + DECLARE_EVENT_CLASS(EvCancelImportRequest) { + TEvCancelImportRequest() = default; + + explicit TEvCancelImportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrImport::TCancelImportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvCancelImportRequest( + const ui64 txId, + const TString& dbName, + const ui64 importId + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(importId); + } + }; + + DECLARE_EVENT_CLASS(EvCancelImportResponse) { + TEvCancelImportResponse() = default; + + explicit TEvCancelImportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvForgetImportRequest) { + TEvForgetImportRequest() = default; + + explicit TEvForgetImportRequest( + const ui64 txId, + const TString& dbName, + const NKikimrImport::TForgetImportRequest& request + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvForgetImportRequest( + const ui64 txId, + const TString& dbName, + const ui64 importId + ) { + Record.SetTxId(txId); + Record.SetDatabaseName(dbName); + Record.MutableRequest()->SetId(importId); + } + }; + + DECLARE_EVENT_CLASS(EvForgetImportResponse) { + TEvForgetImportResponse() = default; + + explicit TEvForgetImportResponse(const ui64 txId) { + Record.SetTxId(txId); + } + }; + + DECLARE_EVENT_CLASS(EvListImportsRequest) { + TEvListImportsRequest() = default; + + explicit TEvListImportsRequest(const TString& dbName, const NKikimrImport::TListImportsRequest& request) { + Record.SetDatabaseName(dbName); + Record.MutableRequest()->CopyFrom(request); + } + + explicit TEvListImportsRequest( + const TString& dbName, + const ui64 pageSize, + const TString& pageToken, + const TString& kind + ) { + Record.SetDatabaseName(dbName); + + auto& request = *Record.MutableRequest(); + request.SetPageSize(pageSize); + request.SetPageToken(pageToken); + request.SetKind(kind); + } + }; + + DECLARE_EVENT_CLASS(EvListImportsResponse) { + }; + +#undef DECLARE_EVENT_CLASS + +}; // TEvImport + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import__cancel.cpp b/ydb/core/tx/schemeshard/schemeshard_import__cancel.cpp index 5943f85f9d0..bd1d9972c55 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__cancel.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__cancel.cpp @@ -1,210 +1,210 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_import_flow_proposals.h" -#include "schemeshard_import.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_import_flow_proposals.h" +#include "schemeshard_import.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/ptr.h> -#include <util/generic/xrange.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> +#include <util/generic/xrange.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TImport::TTxCancel: public TSchemeShard::TXxport::TTxBase { - TEvImport::TEvCancelImportRequest::TPtr Request; - - explicit TTxCancel(TSelf *self, TEvImport::TEvCancelImportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CANCEL_IMPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - auto respond = [this, &request](Ydb::StatusIds::StatusCode status) -> bool { - auto response = MakeHolder<TEvImport::TEvCancelImportResponse>(request.GetTxId()); - auto& proto = *response->Record.MutableResponse(); - - proto.SetStatus(status); - Send(Request->Sender, std::move(response), 0, Request->Cookie); - - return true; - }; - - auto it = Self->Imports.find(request.GetRequest().GetId()); - if (it == Self->Imports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { - return respond(Ydb::StatusIds::NOT_FOUND); - } - - TImportInfo::TPtr importInfo = it->second; - NIceDb::TNiceDb db(txc.DB); - - switch (importInfo->State) { - case TImportInfo::EState::Done: - case TImportInfo::EState::Cancelled: - return respond(Ydb::StatusIds::SUCCESS); - - case TImportInfo::EState::Waiting: - case TImportInfo::EState::Cancellation: - importInfo->Issue = "Cancelled manually"; - importInfo->State = TImportInfo::EState::Cancelled; - - for (ui32 itemIdx : xrange(importInfo->Items.size())) { - const auto& item = importInfo->Items.at(itemIdx); - - switch (item.State) { - case TImportInfo::EState::Transferring: - if (item.WaitTxId != InvalidTxId) { - importInfo->State = TImportInfo::EState::Cancellation; - Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); - } else if (item.SubState == TImportInfo::TItem::ESubState::Proposed) { - importInfo->State = TImportInfo::EState::Cancellation; - } - break; - - case TImportInfo::EState::BuildIndexes: - if (item.WaitTxId != InvalidTxId) { - importInfo->State = TImportInfo::EState::Cancellation; - Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); - } else if (item.SubState == TImportInfo::TItem::ESubState::Proposed) { - importInfo->State = TImportInfo::EState::Cancellation; - } - break; - - default: - break; - } - } - - Self->PersistImportState(db, importInfo); - SendNotificationsIfFinished(importInfo); - return respond(Ydb::StatusIds::SUCCESS); - - default: - return respond(Ydb::StatusIds::UNDETERMINED); - } - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxCancel - + TEvImport::TEvCancelImportRequest::TPtr Request; + + explicit TTxCancel(TSelf *self, TEvImport::TEvCancelImportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CANCEL_IMPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + auto respond = [this, &request](Ydb::StatusIds::StatusCode status) -> bool { + auto response = MakeHolder<TEvImport::TEvCancelImportResponse>(request.GetTxId()); + auto& proto = *response->Record.MutableResponse(); + + proto.SetStatus(status); + Send(Request->Sender, std::move(response), 0, Request->Cookie); + + return true; + }; + + auto it = Self->Imports.find(request.GetRequest().GetId()); + if (it == Self->Imports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { + return respond(Ydb::StatusIds::NOT_FOUND); + } + + TImportInfo::TPtr importInfo = it->second; + NIceDb::TNiceDb db(txc.DB); + + switch (importInfo->State) { + case TImportInfo::EState::Done: + case TImportInfo::EState::Cancelled: + return respond(Ydb::StatusIds::SUCCESS); + + case TImportInfo::EState::Waiting: + case TImportInfo::EState::Cancellation: + importInfo->Issue = "Cancelled manually"; + importInfo->State = TImportInfo::EState::Cancelled; + + for (ui32 itemIdx : xrange(importInfo->Items.size())) { + const auto& item = importInfo->Items.at(itemIdx); + + switch (item.State) { + case TImportInfo::EState::Transferring: + if (item.WaitTxId != InvalidTxId) { + importInfo->State = TImportInfo::EState::Cancellation; + Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); + } else if (item.SubState == TImportInfo::TItem::ESubState::Proposed) { + importInfo->State = TImportInfo::EState::Cancellation; + } + break; + + case TImportInfo::EState::BuildIndexes: + if (item.WaitTxId != InvalidTxId) { + importInfo->State = TImportInfo::EState::Cancellation; + Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); + } else if (item.SubState == TImportInfo::TItem::ESubState::Proposed) { + importInfo->State = TImportInfo::EState::Cancellation; + } + break; + + default: + break; + } + } + + Self->PersistImportState(db, importInfo); + SendNotificationsIfFinished(importInfo); + return respond(Ydb::StatusIds::SUCCESS); + + default: + return respond(Ydb::StatusIds::UNDETERMINED); + } + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxCancel + struct TSchemeShard::TImport::TTxCancelAck: public TSchemeShard::TXxport::TTxBase { TEvSchemeShard::TEvCancelTxResult::TPtr CancelTxResult = nullptr; - TEvIndexBuilder::TEvCancelResponse::TPtr CancelIndexBuildResult = nullptr; - + TEvIndexBuilder::TEvCancelResponse::TPtr CancelIndexBuildResult = nullptr; + explicit TTxCancelAck(TSelf *self, TEvSchemeShard::TEvCancelTxResult::TPtr& ev) - : TXxport::TTxBase(self) - , CancelTxResult(ev) - { - } - - explicit TTxCancelAck(TSelf *self, TEvIndexBuilder::TEvCancelResponse::TPtr& ev) - : TXxport::TTxBase(self) - , CancelIndexBuildResult(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CANCEL_IMPORT_ACK; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - TTxId txId; - ui64 id; - if (CancelTxResult) { - txId = TTxId(CancelTxResult->Get()->Record.GetTargetTxId()); - id = CancelTxResult->Cookie; - } else if (CancelIndexBuildResult) { - txId = TTxId(CancelIndexBuildResult->Get()->Record.GetTxId()); - id = CancelIndexBuildResult->Cookie; - } else { - Y_FAIL("unreachable"); - } - - if (!Self->Imports.contains(id)) { - return true; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(id); - NIceDb::TNiceDb db(txc.DB); - - if (importInfo->State != TImportInfo::EState::Cancellation) { - return true; - } - - bool found = false; - ui32 itemIdx; - ui32 cancelledItems = 0; - ui32 cancellableItems = 0; - - for (ui32 i : xrange(importInfo->Items.size())) { - auto& item = importInfo->Items.at(i); - - if (item.State != TImportInfo::EState::Transferring && item.State != TImportInfo::EState::BuildIndexes) { - continue; - } - - if (item.WaitTxId != InvalidTxId) { - ++cancellableItems; - } - - if (item.WaitTxId == txId) { - found = true; - - item.State = TImportInfo::EState::Cancelled; - itemIdx = i; - } - - if (item.State == TImportInfo::EState::Cancelled) { - ++cancelledItems; - } - } - - if (!found) { - return true; - } - - Self->TxIdToImport.erase(txId); - Self->PersistImportItemState(db, importInfo, itemIdx); - - if (cancelledItems != cancellableItems) { - return true; - } - - importInfo->State = TImportInfo::EState::Cancelled; - Self->PersistImportState(db, importInfo); - - SendNotificationsIfFinished(importInfo); - return true; - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxCancelAck - + : TXxport::TTxBase(self) + , CancelTxResult(ev) + { + } + + explicit TTxCancelAck(TSelf *self, TEvIndexBuilder::TEvCancelResponse::TPtr& ev) + : TXxport::TTxBase(self) + , CancelIndexBuildResult(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CANCEL_IMPORT_ACK; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + TTxId txId; + ui64 id; + if (CancelTxResult) { + txId = TTxId(CancelTxResult->Get()->Record.GetTargetTxId()); + id = CancelTxResult->Cookie; + } else if (CancelIndexBuildResult) { + txId = TTxId(CancelIndexBuildResult->Get()->Record.GetTxId()); + id = CancelIndexBuildResult->Cookie; + } else { + Y_FAIL("unreachable"); + } + + if (!Self->Imports.contains(id)) { + return true; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(id); + NIceDb::TNiceDb db(txc.DB); + + if (importInfo->State != TImportInfo::EState::Cancellation) { + return true; + } + + bool found = false; + ui32 itemIdx; + ui32 cancelledItems = 0; + ui32 cancellableItems = 0; + + for (ui32 i : xrange(importInfo->Items.size())) { + auto& item = importInfo->Items.at(i); + + if (item.State != TImportInfo::EState::Transferring && item.State != TImportInfo::EState::BuildIndexes) { + continue; + } + + if (item.WaitTxId != InvalidTxId) { + ++cancellableItems; + } + + if (item.WaitTxId == txId) { + found = true; + + item.State = TImportInfo::EState::Cancelled; + itemIdx = i; + } + + if (item.State == TImportInfo::EState::Cancelled) { + ++cancelledItems; + } + } + + if (!found) { + return true; + } + + Self->TxIdToImport.erase(txId); + Self->PersistImportItemState(db, importInfo, itemIdx); + + if (cancelledItems != cancellableItems) { + return true; + } + + importInfo->State = TImportInfo::EState::Cancelled; + Self->PersistImportState(db, importInfo); + + SendNotificationsIfFinished(importInfo); + return true; + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxCancelAck + ITransaction* TSchemeShard::CreateTxCancelImport(TEvImport::TEvCancelImportRequest::TPtr& ev) { - return new TImport::TTxCancel(this, ev); -} - + return new TImport::TTxCancel(this, ev); +} + ITransaction* TSchemeShard::CreateTxCancelImportAck(TEvSchemeShard::TEvCancelTxResult::TPtr& ev) { - return new TImport::TTxCancelAck(this, ev); -} - + return new TImport::TTxCancelAck(this, ev); +} + ITransaction* TSchemeShard::CreateTxCancelImportAck(TEvIndexBuilder::TEvCancelResponse::TPtr& ev) { - return new TImport::TTxCancelAck(this, ev); -} - + return new TImport::TTxCancelAck(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 7a88d22e9e7..947870d51bc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1,1005 +1,1005 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_import_flow_proposals.h" -#include "schemeshard_import_scheme_getter.h" -#include "schemeshard_import_helpers.h" -#include "schemeshard_import.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_import_flow_proposals.h" +#include "schemeshard_import_scheme_getter.h" +#include "schemeshard_import_helpers.h" +#include "schemeshard_import.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_import.pb.h> #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/algorithm.h> -#include <util/generic/ptr.h> -#include <util/generic/xrange.h> -#include <util/string/builder.h> - -namespace NKikimr { + +#include <util/generic/algorithm.h> +#include <util/generic/ptr.h> +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase { - TEvImport::TEvCreateImportRequest::TPtr Request; - bool Progress; - - explicit TTxCreate(TSelf* self, TEvImport::TEvCreateImportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - , Progress(false) - { - } - - TTxType GetTxType() const override { - return TXTYPE_CREATE_IMPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - LOG_D("TImport::TTxCreate: DoExecute"); - LOG_T("Message:\n" << request.ShortDebugString()); - - auto response = MakeHolder<TEvImport::TEvCreateImportResponse>(request.GetTxId()); - - const ui64 id = request.GetTxId(); - if (Self->Imports.contains(id)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Import with id '" << id << "' already exists" - ); - } - - const TString& uid = GetUid(request.GetRequest().GetOperationParams().labels()); - if (uid && Self->ImportsByUid.contains(uid)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Import with uid '" << uid << "' already exists" - ); - } - - const TPath domainPath = TPath::Resolve(request.GetDatabaseName(), Self); - { - TPath::TChecker checks = domainPath.Check(); - checks - .IsResolved() - .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath() - .IsLikeDirectory(); - - if (!checks) { - TString explain; - checks.GetStatus(&explain); - - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed database check: " << explain - ); - } - } - - TImportInfo::TPtr importInfo = nullptr; - - switch (request.GetRequest().GetSettingsCase()) { - case NKikimrImport::TCreateImportRequest::kImportFromS3Settings: - { - auto settings = request.GetRequest().GetImportFromS3Settings(); - if (!settings.scheme()) { - settings.set_scheme(Ydb::Import::ImportFromS3Settings::HTTPS); - } - - importInfo = new TImportInfo(id, uid, TImportInfo::EKind::S3, settings, domainPath.Base()->PathId); - - if (request.HasUserSID()) { - importInfo->UserSID = request.GetUserSID(); - } - - TString explain; - if (!FillItems(importInfo, settings, explain)) { - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed item check: " << explain - ); - } - } - break; - - default: - Y_VERIFY_DEBUG(false, "Unknown import kind"); - } - - Y_VERIFY(importInfo != nullptr); - - NIceDb::TNiceDb db(txc.DB); - Self->PersistCreateImport(db, importInfo); - - importInfo->State = TImportInfo::EState::Waiting; - Self->PersistImportState(db, importInfo); - - Self->Imports[id] = importInfo; - if (uid) { - Self->ImportsByUid[uid] = importInfo; - } - - Self->FromXxportInfo(*response->Record.MutableResponse()->MutableEntry(), importInfo); - - Progress = true; - return Reply(std::move(response)); - } - - void DoComplete(const TActorContext& ctx) override { - LOG_D("TImport::TTxCreate: DoComplete"); - - if (Progress) { - const ui64 id = Request->Get()->Record.GetTxId(); - Self->Execute(Self->CreateTxProgressImport(id), ctx); - } - } - -private: - static TString GetUid(const google::protobuf::Map<TString, TString>& labels) { - auto it = labels.find("uid"); - if (it == labels.end()) { - return TString(); - } - - return it->second; - } - - bool Reply( - THolder<TEvImport::TEvCreateImportResponse> response, - const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS, - const TString& errorMessage = TString() - ) { - LOG_D("TImport::TTxCreate: Reply" - << ": status# " << status - << ", error# " << errorMessage); - LOG_T("Message:\n" << response->Record.ShortDebugString()); - - auto& entry = *response->Record.MutableResponse()->MutableEntry(); - entry.SetStatus(status); - if (errorMessage) { - AddIssue(entry, errorMessage); - } - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - - return true; - } - - template <typename TSettings> - bool FillItems(TImportInfo::TPtr importInfo, const TSettings& settings, TString& explain) { - importInfo->Items.reserve(settings.items().size()); - for (ui32 itemIdx : xrange(settings.items().size())) { - const auto& item = settings.items(itemIdx); - const TPath path = TPath::Resolve(item.destination_path(), Self); - { - TPath::TChecker checks = path.Check(); - checks - .IsAtLocalSchemeShard() - .IsValidLeafName() - .DepthLimit() - .PathsLimit(); - - if (path.IsResolved()) { - checks - .IsResolved() - .IsDeleted(); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - if (path.Parent().IsResolved()) { - checks.DirChildrenLimit(); - } - - if (!checks) { - checks.GetStatus(&explain); - return false; - } - } - - importInfo->Items.emplace_back(item.destination_path()); - } - - return true; - } - -}; // TTxCreate - + TEvImport::TEvCreateImportRequest::TPtr Request; + bool Progress; + + explicit TTxCreate(TSelf* self, TEvImport::TEvCreateImportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + , Progress(false) + { + } + + TTxType GetTxType() const override { + return TXTYPE_CREATE_IMPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + LOG_D("TImport::TTxCreate: DoExecute"); + LOG_T("Message:\n" << request.ShortDebugString()); + + auto response = MakeHolder<TEvImport::TEvCreateImportResponse>(request.GetTxId()); + + const ui64 id = request.GetTxId(); + if (Self->Imports.contains(id)) { + return Reply( + std::move(response), + Ydb::StatusIds::ALREADY_EXISTS, + TStringBuilder() << "Import with id '" << id << "' already exists" + ); + } + + const TString& uid = GetUid(request.GetRequest().GetOperationParams().labels()); + if (uid && Self->ImportsByUid.contains(uid)) { + return Reply( + std::move(response), + Ydb::StatusIds::ALREADY_EXISTS, + TStringBuilder() << "Import with uid '" << uid << "' already exists" + ); + } + + const TPath domainPath = TPath::Resolve(request.GetDatabaseName(), Self); + { + TPath::TChecker checks = domainPath.Check(); + checks + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory(); + + if (!checks) { + TString explain; + checks.GetStatus(&explain); + + return Reply( + std::move(response), + Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Failed database check: " << explain + ); + } + } + + TImportInfo::TPtr importInfo = nullptr; + + switch (request.GetRequest().GetSettingsCase()) { + case NKikimrImport::TCreateImportRequest::kImportFromS3Settings: + { + auto settings = request.GetRequest().GetImportFromS3Settings(); + if (!settings.scheme()) { + settings.set_scheme(Ydb::Import::ImportFromS3Settings::HTTPS); + } + + importInfo = new TImportInfo(id, uid, TImportInfo::EKind::S3, settings, domainPath.Base()->PathId); + + if (request.HasUserSID()) { + importInfo->UserSID = request.GetUserSID(); + } + + TString explain; + if (!FillItems(importInfo, settings, explain)) { + return Reply( + std::move(response), + Ydb::StatusIds::BAD_REQUEST, + TStringBuilder() << "Failed item check: " << explain + ); + } + } + break; + + default: + Y_VERIFY_DEBUG(false, "Unknown import kind"); + } + + Y_VERIFY(importInfo != nullptr); + + NIceDb::TNiceDb db(txc.DB); + Self->PersistCreateImport(db, importInfo); + + importInfo->State = TImportInfo::EState::Waiting; + Self->PersistImportState(db, importInfo); + + Self->Imports[id] = importInfo; + if (uid) { + Self->ImportsByUid[uid] = importInfo; + } + + Self->FromXxportInfo(*response->Record.MutableResponse()->MutableEntry(), importInfo); + + Progress = true; + return Reply(std::move(response)); + } + + void DoComplete(const TActorContext& ctx) override { + LOG_D("TImport::TTxCreate: DoComplete"); + + if (Progress) { + const ui64 id = Request->Get()->Record.GetTxId(); + Self->Execute(Self->CreateTxProgressImport(id), ctx); + } + } + +private: + static TString GetUid(const google::protobuf::Map<TString, TString>& labels) { + auto it = labels.find("uid"); + if (it == labels.end()) { + return TString(); + } + + return it->second; + } + + bool Reply( + THolder<TEvImport::TEvCreateImportResponse> response, + const Ydb::StatusIds::StatusCode status = Ydb::StatusIds::SUCCESS, + const TString& errorMessage = TString() + ) { + LOG_D("TImport::TTxCreate: Reply" + << ": status# " << status + << ", error# " << errorMessage); + LOG_T("Message:\n" << response->Record.ShortDebugString()); + + auto& entry = *response->Record.MutableResponse()->MutableEntry(); + entry.SetStatus(status); + if (errorMessage) { + AddIssue(entry, errorMessage); + } + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + + return true; + } + + template <typename TSettings> + bool FillItems(TImportInfo::TPtr importInfo, const TSettings& settings, TString& explain) { + importInfo->Items.reserve(settings.items().size()); + for (ui32 itemIdx : xrange(settings.items().size())) { + const auto& item = settings.items(itemIdx); + const TPath path = TPath::Resolve(item.destination_path(), Self); + { + TPath::TChecker checks = path.Check(); + checks + .IsAtLocalSchemeShard() + .IsValidLeafName() + .DepthLimit() + .PathsLimit(); + + if (path.IsResolved()) { + checks + .IsResolved() + .IsDeleted(); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (path.Parent().IsResolved()) { + checks.DirChildrenLimit(); + } + + if (!checks) { + checks.GetStatus(&explain); + return false; + } + } + + importInfo->Items.emplace_back(item.destination_path()); + } + + return true; + } + +}; // TTxCreate + struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase { - using EState = TImportInfo::EState; - using ESubState = TImportInfo::TItem::ESubState; - - static constexpr ui32 IssuesSizeLimit = 2 * 1024; - - ui64 Id; - TEvPrivate::TEvImportSchemeReady::TPtr SchemeResult = nullptr; - TEvTxAllocatorClient::TEvAllocateResult::TPtr AllocateResult = nullptr; + using EState = TImportInfo::EState; + using ESubState = TImportInfo::TItem::ESubState; + + static constexpr ui32 IssuesSizeLimit = 2 * 1024; + + ui64 Id; + TEvPrivate::TEvImportSchemeReady::TPtr SchemeResult = nullptr; + TEvTxAllocatorClient::TEvAllocateResult::TPtr AllocateResult = nullptr; TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr ModifyResult = nullptr; - TEvIndexBuilder::TEvCreateResponse::TPtr CreateIndexResult = nullptr; + TEvIndexBuilder::TEvCreateResponse::TPtr CreateIndexResult = nullptr; TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr NotifyResult = nullptr; - - explicit TTxProgress(TSelf* self, ui64 id) - : TXxport::TTxBase(self) - , Id(id) - { - } - - explicit TTxProgress(TSelf* self, TEvPrivate::TEvImportSchemeReady::TPtr& ev) - : TXxport::TTxBase(self) - , SchemeResult(ev) - { - } - - explicit TTxProgress(TSelf* self, TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) - : TXxport::TTxBase(self) - , AllocateResult(ev) - { - } - + + explicit TTxProgress(TSelf* self, ui64 id) + : TXxport::TTxBase(self) + , Id(id) + { + } + + explicit TTxProgress(TSelf* self, TEvPrivate::TEvImportSchemeReady::TPtr& ev) + : TXxport::TTxBase(self) + , SchemeResult(ev) + { + } + + explicit TTxProgress(TSelf* self, TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) + : TXxport::TTxBase(self) + , AllocateResult(ev) + { + } + explicit TTxProgress(TSelf* self, TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) - : TXxport::TTxBase(self) - , ModifyResult(ev) - { - } - - explicit TTxProgress(TSelf* self, TEvIndexBuilder::TEvCreateResponse::TPtr& ev) - : TXxport::TTxBase(self) - , CreateIndexResult(ev) - { - } - + : TXxport::TTxBase(self) + , ModifyResult(ev) + { + } + + explicit TTxProgress(TSelf* self, TEvIndexBuilder::TEvCreateResponse::TPtr& ev) + : TXxport::TTxBase(self) + , CreateIndexResult(ev) + { + } + explicit TTxProgress(TSelf* self, TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) - : TXxport::TTxBase(self) - , NotifyResult(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_IMPORT_PROGRESS; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - LOG_D("TImport::TTxProgress: DoExecute"); - - if (SchemeResult) { - OnSchemeResult(txc, ctx); - } else if (AllocateResult) { - OnAllocateResult(txc, ctx); - } else if (ModifyResult) { - OnModifyResult(txc, ctx); - } else if (CreateIndexResult) { - OnCreateIndexResult(txc, ctx); - } else if (NotifyResult) { - OnNotifyResult(txc, ctx); - } else { - Resume(txc, ctx); - } - - return true; - } - - void DoComplete(const TActorContext&) override { - LOG_D("TImport::TTxProgress: DoComplete"); - } - -private: - void GetScheme(TImportInfo::TPtr importInfo, ui32 itemIdx, const TActorContext& ctx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - LOG_I("TImport::TTxProgress: Get scheme" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - + : TXxport::TTxBase(self) + , NotifyResult(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_IMPORT_PROGRESS; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + LOG_D("TImport::TTxProgress: DoExecute"); + + if (SchemeResult) { + OnSchemeResult(txc, ctx); + } else if (AllocateResult) { + OnAllocateResult(txc, ctx); + } else if (ModifyResult) { + OnModifyResult(txc, ctx); + } else if (CreateIndexResult) { + OnCreateIndexResult(txc, ctx); + } else if (NotifyResult) { + OnNotifyResult(txc, ctx); + } else { + Resume(txc, ctx); + } + + return true; + } + + void DoComplete(const TActorContext&) override { + LOG_D("TImport::TTxProgress: DoComplete"); + } + +private: + void GetScheme(TImportInfo::TPtr importInfo, ui32 itemIdx, const TActorContext& ctx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + LOG_I("TImport::TTxProgress: Get scheme" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + ctx.RegisterWithSameMailbox(CreateSchemeGetter(Self->SelfId(), importInfo, itemIdx)); - } - - void CreateTable(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::Proposed; - - LOG_I("TImport::TTxProgress: CreateTable propose" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - - auto propose = CreateTablePropose(Self, txId, importInfo, itemIdx); - Y_VERIFY(propose); - - Send(Self->SelfId(), std::move(propose)); - } - - void TransferData(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::Proposed; - - LOG_I("TImport::TTxProgress: Restore propose" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->SelfId(), RestorePropose(Self, txId, importInfo, itemIdx)); - } - - bool CancelTransferring(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - if (item.WaitTxId == InvalidTxId) { - if (item.SubState == ESubState::Proposed) { - importInfo->State = EState::Cancellation; - } - - return false; - } - - importInfo->State = EState::Cancellation; - - LOG_I("TImport::TTxProgress: cancel restore's tx" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); - return true; - } - - void BuildIndex(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::Proposed; - - LOG_I("TImport::TTxProgress: build index" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->SelfId(), BuildIndexPropose(Self, txId, importInfo, itemIdx, MakeIndexBuildUid(importInfo, itemIdx))); - } - - bool CancelIndexBuilding(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - if (item.WaitTxId == InvalidTxId) { - if (item.SubState == ESubState::Proposed) { - importInfo->State = EState::Cancellation; - } - - return false; - } - - importInfo->State = EState::Cancellation; - - LOG_I("TImport::TTxProgress: cancel index building" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); - return true; - } - - void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::AllocateTxId; - - LOG_I("TImport::TTxProgress: Allocate txId" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Y_VERIFY(item.WaitTxId == InvalidTxId); - Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, importInfo->Id); - } - - void SubscribeTx(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::Subscribed; - - LOG_I("TImport::TTxProgress: Wait for completion" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Y_VERIFY(item.WaitTxId != InvalidTxId); + } + + void CreateTable(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TImport::TTxProgress: CreateTable propose" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + + auto propose = CreateTablePropose(Self, txId, importInfo, itemIdx); + Y_VERIFY(propose); + + Send(Self->SelfId(), std::move(propose)); + } + + void TransferData(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TImport::TTxProgress: Restore propose" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->SelfId(), RestorePropose(Self, txId, importInfo, itemIdx)); + } + + bool CancelTransferring(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + if (item.WaitTxId == InvalidTxId) { + if (item.SubState == ESubState::Proposed) { + importInfo->State = EState::Cancellation; + } + + return false; + } + + importInfo->State = EState::Cancellation; + + LOG_I("TImport::TTxProgress: cancel restore's tx" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); + return true; + } + + void BuildIndex(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TImport::TTxProgress: build index" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->SelfId(), BuildIndexPropose(Self, txId, importInfo, itemIdx, MakeIndexBuildUid(importInfo, itemIdx))); + } + + bool CancelIndexBuilding(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + if (item.WaitTxId == InvalidTxId) { + if (item.SubState == ESubState::Proposed) { + importInfo->State = EState::Cancellation; + } + + return false; + } + + importInfo->State = EState::Cancellation; + + LOG_I("TImport::TTxProgress: cancel index building" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); + return true; + } + + void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::AllocateTxId; + + LOG_I("TImport::TTxProgress: Allocate txId" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Y_VERIFY(item.WaitTxId == InvalidTxId); + Send(Self->TxAllocatorClient, new TEvTxAllocatorClient::TEvAllocate(), 0, importInfo->Id); + } + + void SubscribeTx(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Subscribed; + + LOG_I("TImport::TTxProgress: Wait for completion" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Y_VERIFY(item.WaitTxId != InvalidTxId); Send(Self->SelfId(), new TEvSchemeShard::TEvNotifyTxCompletion(ui64(item.WaitTxId))); - } - - TTxId GetActiveRestoreTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - Y_VERIFY(item.State == EState::Transferring); - Y_VERIFY(item.DstPathId); - - if (!Self->PathsById.contains(item.DstPathId)) { - return InvalidTxId; - } - - auto path = Self->PathsById.at(item.DstPathId); + } + + TTxId GetActiveRestoreTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + Y_VERIFY(item.State == EState::Transferring); + Y_VERIFY(item.DstPathId); + + if (!Self->PathsById.contains(item.DstPathId)) { + return InvalidTxId; + } + + auto path = Self->PathsById.at(item.DstPathId); if (path->PathState != NKikimrSchemeOp::EPathStateRestore) { - return InvalidTxId; - } - - return path->LastTxId; - } - - TTxId GetActiveBuildIndexId(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - Y_VERIFY(item.State == EState::BuildIndexes); - - const auto uid = MakeIndexBuildUid(importInfo, itemIdx); - if (!Self->IndexBuildsByUid.contains(uid)) { - return InvalidTxId; - } - - return TTxId(ui64(Self->IndexBuildsByUid.at(uid)->Id)); - } - - static TString MakeIndexBuildUid(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - return TStringBuilder() << importInfo->Id << "-" << itemIdx << "-" << item.NextIndexIdx; - } - - void Cancel(TImportInfo::TPtr importInfo, ui32 itemIdx, TStringBuf marker) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - LOG_N("TImport::TTxProgress: " << marker << ", cancelling" - << ", info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - importInfo->State = EState::Cancelled; - - for (ui32 i : xrange(importInfo->Items.size())) { - if (i == itemIdx) { - continue; - } - - switch (importInfo->Items.at(i).State) { - case EState::Transferring: - CancelTransferring(importInfo, i); - break; - - case EState::BuildIndexes: - CancelIndexBuilding(importInfo, i); - break; - - default: - break; - } - } - } - - TMaybe<TString> GetIssues(const TPathId& dstPathId, TTxId restoreTxId) { - Y_VERIFY(Self->Tables.contains(dstPathId)); - TTableInfo::TPtr table = Self->Tables.at(dstPathId); - - Y_VERIFY(table->RestoreHistory.contains(restoreTxId)); - const auto& result = table->RestoreHistory.at(restoreTxId); - - if (result.TotalShardCount == result.SuccessShardCount) { - return Nothing(); - } - - TStringBuilder output; - bool first = true; - - for (const auto& [shardId, status] : result.ShardStatuses) { - if (status.Success) { - continue; - } - - if (output.size() > IssuesSizeLimit) { - output << "... <truncated>"; - break; - } - - if (!first) { - output << ", "; - } - first = false; - - output << "shard: " << shardId << ", error: " << status.Error; - } - - return output; - } - - TMaybe<TString> GetIssues(TIndexBuildId indexBuildId) { - Y_VERIFY(Self->IndexBuilds.contains(indexBuildId)); - TIndexBuildInfo::TPtr indexInfo = Self->IndexBuilds.at(indexBuildId); - - if (indexInfo->IsDone()) { - return Nothing(); - } - - return indexInfo->Issue; - } - - TString GetIssues(const NKikimrIndexBuilder::TEvCreateResponse& proto) { - TStringBuilder output; - - for (const auto& issue : proto.GetIssues()) { - output << issue.message() << Endl; - } - - return output; - } - - void Resume(TTransactionContext& txc, const TActorContext& ctx) { - Y_VERIFY(Self->Imports.contains(Id)); - TImportInfo::TPtr importInfo = Self->Imports.at(Id); - - LOG_D("TImport::TTxProgress: Resume" - << ": id# " << Id); - - NIceDb::TNiceDb db(txc.DB); - - switch (importInfo->State) { - case EState::Waiting: - for (ui32 itemIdx : xrange(importInfo->Items.size())) { - const auto& item = importInfo->Items.at(itemIdx); - - switch (item.State) { - case EState::GetScheme: - GetScheme(importInfo, itemIdx, ctx); - break; - - case EState::CreateTable: - case EState::Transferring: - case EState::BuildIndexes: - if (item.WaitTxId == InvalidTxId) { - AllocateTxId(importInfo, itemIdx); - } else { - SubscribeTx(importInfo, itemIdx); - } - break; - - default: - break; - } - } - break; - - case EState::Cancellation: - for (ui32 itemIdx : xrange(importInfo->Items.size())) { - auto& item = importInfo->Items.at(itemIdx); - TTxId txId = InvalidTxId; - - switch (item.State) { - case EState::Transferring: - if (!CancelTransferring(importInfo, itemIdx)) { - txId = GetActiveRestoreTxId(importInfo, itemIdx); - } - break; - - case EState::BuildIndexes: - if (!CancelIndexBuilding(importInfo, itemIdx)) { - txId = GetActiveBuildIndexId(importInfo, itemIdx); - } - break; - - default: - break; - } - - if (txId != InvalidTxId) { - item.WaitTxId = txId; - Self->PersistImportItemState(db, importInfo, itemIdx); - - switch (item.State) { - case EState::Transferring: - CancelTransferring(importInfo, itemIdx); - break; - - case EState::BuildIndexes: - CancelIndexBuilding(importInfo, itemIdx); - break; - - default: - break; - } - } - } - break; - - default: - break; - } - } - - void OnSchemeResult(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(SchemeResult); - - const auto& msg = *SchemeResult->Get(); - - LOG_D("TImport::TTxProgress: OnSchemeResult" - << ": id# " << msg.ImportId); - - if (!Self->Imports.contains(msg.ImportId)) { - LOG_E("TImport::TTxProgress: OnSchemeResult received unknown id" - << ": id# " << msg.ImportId); - return; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(msg.ImportId); - if (msg.ItemIdx >= importInfo->Items.size()) { - LOG_E("TImport::TTxProgress: OnSchemeResult received unknown item" - << ": id# " << msg.ImportId - << ", item# " << msg.ItemIdx); - return; - } - - auto& item = importInfo->Items.at(msg.ItemIdx); - NIceDb::TNiceDb db(txc.DB); - - TString error; - if (!msg.Success || !CreateTablePropose(Self, TTxId(), importInfo, msg.ItemIdx, error)) { - item.Issue = msg.Success ? error : msg.Error; - Self->PersistImportItemState(db, importInfo, msg.ItemIdx); - - if (importInfo->State != EState::Waiting) { - return; - } - - Cancel(importInfo, msg.ItemIdx, "cannot get/invalid scheme"); - Self->PersistImportState(db, importInfo); - - return SendNotificationsIfFinished(importInfo); - } - - Self->PersistImportItemScheme(db, importInfo, msg.ItemIdx); - - item.State = EState::CreateTable; - Self->PersistImportItemState(db, importInfo, msg.ItemIdx); - AllocateTxId(importInfo, msg.ItemIdx); - } - - void OnAllocateResult(TTransactionContext&, const TActorContext&) { - Y_VERIFY(AllocateResult); - + return InvalidTxId; + } + + return path->LastTxId; + } + + TTxId GetActiveBuildIndexId(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + Y_VERIFY(item.State == EState::BuildIndexes); + + const auto uid = MakeIndexBuildUid(importInfo, itemIdx); + if (!Self->IndexBuildsByUid.contains(uid)) { + return InvalidTxId; + } + + return TTxId(ui64(Self->IndexBuildsByUid.at(uid)->Id)); + } + + static TString MakeIndexBuildUid(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + return TStringBuilder() << importInfo->Id << "-" << itemIdx << "-" << item.NextIndexIdx; + } + + void Cancel(TImportInfo::TPtr importInfo, ui32 itemIdx, TStringBuf marker) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + LOG_N("TImport::TTxProgress: " << marker << ", cancelling" + << ", info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + importInfo->State = EState::Cancelled; + + for (ui32 i : xrange(importInfo->Items.size())) { + if (i == itemIdx) { + continue; + } + + switch (importInfo->Items.at(i).State) { + case EState::Transferring: + CancelTransferring(importInfo, i); + break; + + case EState::BuildIndexes: + CancelIndexBuilding(importInfo, i); + break; + + default: + break; + } + } + } + + TMaybe<TString> GetIssues(const TPathId& dstPathId, TTxId restoreTxId) { + Y_VERIFY(Self->Tables.contains(dstPathId)); + TTableInfo::TPtr table = Self->Tables.at(dstPathId); + + Y_VERIFY(table->RestoreHistory.contains(restoreTxId)); + const auto& result = table->RestoreHistory.at(restoreTxId); + + if (result.TotalShardCount == result.SuccessShardCount) { + return Nothing(); + } + + TStringBuilder output; + bool first = true; + + for (const auto& [shardId, status] : result.ShardStatuses) { + if (status.Success) { + continue; + } + + if (output.size() > IssuesSizeLimit) { + output << "... <truncated>"; + break; + } + + if (!first) { + output << ", "; + } + first = false; + + output << "shard: " << shardId << ", error: " << status.Error; + } + + return output; + } + + TMaybe<TString> GetIssues(TIndexBuildId indexBuildId) { + Y_VERIFY(Self->IndexBuilds.contains(indexBuildId)); + TIndexBuildInfo::TPtr indexInfo = Self->IndexBuilds.at(indexBuildId); + + if (indexInfo->IsDone()) { + return Nothing(); + } + + return indexInfo->Issue; + } + + TString GetIssues(const NKikimrIndexBuilder::TEvCreateResponse& proto) { + TStringBuilder output; + + for (const auto& issue : proto.GetIssues()) { + output << issue.message() << Endl; + } + + return output; + } + + void Resume(TTransactionContext& txc, const TActorContext& ctx) { + Y_VERIFY(Self->Imports.contains(Id)); + TImportInfo::TPtr importInfo = Self->Imports.at(Id); + + LOG_D("TImport::TTxProgress: Resume" + << ": id# " << Id); + + NIceDb::TNiceDb db(txc.DB); + + switch (importInfo->State) { + case EState::Waiting: + for (ui32 itemIdx : xrange(importInfo->Items.size())) { + const auto& item = importInfo->Items.at(itemIdx); + + switch (item.State) { + case EState::GetScheme: + GetScheme(importInfo, itemIdx, ctx); + break; + + case EState::CreateTable: + case EState::Transferring: + case EState::BuildIndexes: + if (item.WaitTxId == InvalidTxId) { + AllocateTxId(importInfo, itemIdx); + } else { + SubscribeTx(importInfo, itemIdx); + } + break; + + default: + break; + } + } + break; + + case EState::Cancellation: + for (ui32 itemIdx : xrange(importInfo->Items.size())) { + auto& item = importInfo->Items.at(itemIdx); + TTxId txId = InvalidTxId; + + switch (item.State) { + case EState::Transferring: + if (!CancelTransferring(importInfo, itemIdx)) { + txId = GetActiveRestoreTxId(importInfo, itemIdx); + } + break; + + case EState::BuildIndexes: + if (!CancelIndexBuilding(importInfo, itemIdx)) { + txId = GetActiveBuildIndexId(importInfo, itemIdx); + } + break; + + default: + break; + } + + if (txId != InvalidTxId) { + item.WaitTxId = txId; + Self->PersistImportItemState(db, importInfo, itemIdx); + + switch (item.State) { + case EState::Transferring: + CancelTransferring(importInfo, itemIdx); + break; + + case EState::BuildIndexes: + CancelIndexBuilding(importInfo, itemIdx); + break; + + default: + break; + } + } + } + break; + + default: + break; + } + } + + void OnSchemeResult(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(SchemeResult); + + const auto& msg = *SchemeResult->Get(); + + LOG_D("TImport::TTxProgress: OnSchemeResult" + << ": id# " << msg.ImportId); + + if (!Self->Imports.contains(msg.ImportId)) { + LOG_E("TImport::TTxProgress: OnSchemeResult received unknown id" + << ": id# " << msg.ImportId); + return; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(msg.ImportId); + if (msg.ItemIdx >= importInfo->Items.size()) { + LOG_E("TImport::TTxProgress: OnSchemeResult received unknown item" + << ": id# " << msg.ImportId + << ", item# " << msg.ItemIdx); + return; + } + + auto& item = importInfo->Items.at(msg.ItemIdx); + NIceDb::TNiceDb db(txc.DB); + + TString error; + if (!msg.Success || !CreateTablePropose(Self, TTxId(), importInfo, msg.ItemIdx, error)) { + item.Issue = msg.Success ? error : msg.Error; + Self->PersistImportItemState(db, importInfo, msg.ItemIdx); + + if (importInfo->State != EState::Waiting) { + return; + } + + Cancel(importInfo, msg.ItemIdx, "cannot get/invalid scheme"); + Self->PersistImportState(db, importInfo); + + return SendNotificationsIfFinished(importInfo); + } + + Self->PersistImportItemScheme(db, importInfo, msg.ItemIdx); + + item.State = EState::CreateTable; + Self->PersistImportItemState(db, importInfo, msg.ItemIdx); + AllocateTxId(importInfo, msg.ItemIdx); + } + + void OnAllocateResult(TTransactionContext&, const TActorContext&) { + Y_VERIFY(AllocateResult); + const auto txId = TTxId(AllocateResult->Get()->TxIds.front()); - const ui64 id = AllocateResult->Cookie; - - LOG_D("TImport::TTxProgress: OnAllocateResult" - << ": txId# " << txId - << ", id# " << id); - - if (!Self->Imports.contains(id)) { - LOG_E("TImport::TTxProgress: OnAllocateResult received unknown id" - << ": id# " << id); - return; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(id); - if (importInfo->State != EState::Waiting) { - return; - } - - TMaybe<ui32> itemIdx; - for (ui32 i : xrange(importInfo->Items.size())) { - const auto& item = importInfo->Items.at(i); - - if (item.SubState != ESubState::AllocateTxId) { - continue; - } - - switch (item.State) { - case EState::CreateTable: - CreateTable(importInfo, i, txId); - itemIdx = i; - break; - - case EState::Transferring: - TransferData(importInfo, i, txId); - itemIdx = i; - break; - - case EState::BuildIndexes: - BuildIndex(importInfo, i, txId); - itemIdx = i; - break; - - default: - break; - } - - if (itemIdx) { - break; - } - } - - if (!itemIdx) { - return; - } - - Y_VERIFY(!Self->TxIdToImport.contains(txId)); - Self->TxIdToImport[txId] = {importInfo->Id, *itemIdx}; - } - - void OnModifyResult(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(ModifyResult); - const auto& record = ModifyResult->Get()->Record; - - LOG_D("TImport::TTxProgress: OnModifyResult" - << ": txId# " << record.GetTxId() - << ", status# " << record.GetStatus()); - LOG_T("Message:\n" << record.ShortDebugString()); - - auto txId = TTxId(record.GetTxId()); - if (!Self->TxIdToImport.contains(txId)) { - LOG_E("TImport::TTxProgress: OnModifyResult received unknown txId" - << ": txId# " << txId); - return; - } - - ui64 id; - ui32 itemIdx; - std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); - if (!Self->Imports.contains(id)) { - LOG_E("TImport::TTxProgress: OnModifyResult received unknown id" - << ": id# " << id); - return; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(id); - NIceDb::TNiceDb db(txc.DB); - - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - + const ui64 id = AllocateResult->Cookie; + + LOG_D("TImport::TTxProgress: OnAllocateResult" + << ": txId# " << txId + << ", id# " << id); + + if (!Self->Imports.contains(id)) { + LOG_E("TImport::TTxProgress: OnAllocateResult received unknown id" + << ": id# " << id); + return; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(id); + if (importInfo->State != EState::Waiting) { + return; + } + + TMaybe<ui32> itemIdx; + for (ui32 i : xrange(importInfo->Items.size())) { + const auto& item = importInfo->Items.at(i); + + if (item.SubState != ESubState::AllocateTxId) { + continue; + } + + switch (item.State) { + case EState::CreateTable: + CreateTable(importInfo, i, txId); + itemIdx = i; + break; + + case EState::Transferring: + TransferData(importInfo, i, txId); + itemIdx = i; + break; + + case EState::BuildIndexes: + BuildIndex(importInfo, i, txId); + itemIdx = i; + break; + + default: + break; + } + + if (itemIdx) { + break; + } + } + + if (!itemIdx) { + return; + } + + Y_VERIFY(!Self->TxIdToImport.contains(txId)); + Self->TxIdToImport[txId] = {importInfo->Id, *itemIdx}; + } + + void OnModifyResult(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(ModifyResult); + const auto& record = ModifyResult->Get()->Record; + + LOG_D("TImport::TTxProgress: OnModifyResult" + << ": txId# " << record.GetTxId() + << ", status# " << record.GetStatus()); + LOG_T("Message:\n" << record.ShortDebugString()); + + auto txId = TTxId(record.GetTxId()); + if (!Self->TxIdToImport.contains(txId)) { + LOG_E("TImport::TTxProgress: OnModifyResult received unknown txId" + << ": txId# " << txId); + return; + } + + ui64 id; + ui32 itemIdx; + std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); + if (!Self->Imports.contains(id)) { + LOG_E("TImport::TTxProgress: OnModifyResult received unknown id" + << ": id# " << id); + return; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(id); + NIceDb::TNiceDb db(txc.DB); + + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + if (record.GetStatus() != NKikimrScheme::StatusAccepted) { - Self->TxIdToImport.erase(txId); - txId = InvalidTxId; - + Self->TxIdToImport.erase(txId); + txId = InvalidTxId; + if (record.GetStatus() == NKikimrScheme::StatusMultipleModifications) { - if (record.GetPathCreateTxId()) { - txId = TTxId(record.GetPathCreateTxId()); - } else if (item.State == EState::Transferring) { - txId = GetActiveRestoreTxId(importInfo, itemIdx); - } - } - - if (txId == InvalidTxId) { - item.Issue = record.GetReason(); - Self->PersistImportItemState(db, importInfo, itemIdx); - - if (importInfo->State != EState::Waiting) { - return; - } - - Cancel(importInfo, itemIdx, "unhappy propose"); - Self->PersistImportState(db, importInfo); - - return SendNotificationsIfFinished(importInfo); - } - - Self->TxIdToImport[txId] = {importInfo->Id, itemIdx}; - } - - item.WaitTxId = txId; - Self->PersistImportItemState(db, importInfo, itemIdx); - - if (importInfo->State != EState::Waiting && item.State == EState::Transferring) { - CancelTransferring(importInfo, itemIdx); - return; - } - - if (item.State == EState::CreateTable) { - item.DstPathId = Self->MakeLocalId(TLocalPathId(record.GetPathId())); - Self->PersistImportItemDstPathId(db, importInfo, itemIdx); - } - - SubscribeTx(importInfo, itemIdx); - } - - void OnCreateIndexResult(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(CreateIndexResult); - const auto& record = CreateIndexResult->Get()->Record; - - LOG_D("TImport::TTxProgress: OnCreateIndexResult" - << ": txId# " << record.GetTxId() - << ", status# " << record.GetStatus()); - LOG_T("Message:\n" << record.ShortDebugString()); - - auto txId = TTxId(record.GetTxId()); - if (!Self->TxIdToImport.contains(txId)) { - LOG_E("TImport::TTxProgress: OnCreateIndexResult received unknown txId" - << ": txId# " << txId); - return; - } - - ui64 id; - ui32 itemIdx; - std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); - if (!Self->Imports.contains(id)) { - LOG_E("TImport::TTxProgress: OnCreateIndexResult received unknown id" - << ": id# " << id); - return; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(id); - NIceDb::TNiceDb db(txc.DB); - - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { - Self->TxIdToImport.erase(txId); - txId = InvalidTxId; - - if (record.GetStatus() == Ydb::StatusIds::ALREADY_EXISTS) { - if (item.State == EState::BuildIndexes) { - txId = GetActiveBuildIndexId(importInfo, itemIdx); - } - } - - if (txId == InvalidTxId) { - item.Issue = GetIssues(record); - Self->PersistImportItemState(db, importInfo, itemIdx); - - if (importInfo->State != EState::Waiting) { - return; - } - - Cancel(importInfo, itemIdx, "unhappy propose"); - Self->PersistImportState(db, importInfo); - - return SendNotificationsIfFinished(importInfo); - } - - Self->TxIdToImport[txId] = {importInfo->Id, itemIdx}; - } - - item.WaitTxId = txId; - Self->PersistImportItemState(db, importInfo, itemIdx); - - if (importInfo->State != EState::Waiting) { - CancelIndexBuilding(importInfo, itemIdx); - return; - } - - SubscribeTx(importInfo, itemIdx); - } - - void OnNotifyResult(TTransactionContext& txc, const TActorContext&) { - Y_VERIFY(NotifyResult); - const auto& record = NotifyResult->Get()->Record; - - LOG_D("TImport::TTxProgress: OnNotifyResult" - << ": txId# " << record.GetTxId()); - - const auto txId = TTxId(record.GetTxId()); - if (!Self->TxIdToImport.contains(txId)) { - LOG_E("TImport::TTxProgress: OnNotifyResult received unknown txId" - << ": txId# " << txId); - return; - } - - ui64 id; - ui32 itemIdx; - std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); - if (!Self->Imports.contains(id)) { - LOG_E("TImport::TTxProgress: OnNotifyResult received unknown id" - << ": id# " << id); - return; - } - - TImportInfo::TPtr importInfo = Self->Imports.at(id); - NIceDb::TNiceDb db(txc.DB); - - Y_VERIFY(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.WaitTxId = InvalidTxId; - Self->PersistImportItemState(db, importInfo, itemIdx); - - Self->TxIdToImport.erase(txId); - - if (importInfo->State != EState::Waiting) { - return; - } - - switch (item.State) { - case EState::CreateTable: - item.State = EState::Transferring; - AllocateTxId(importInfo, itemIdx); - break; - - case EState::Transferring: - if (const auto issue = GetIssues(item.DstPathId, txId)) { - item.Issue = *issue; - Cancel(importInfo, itemIdx, "issues during restore"); - } else { - if (item.NextIndexIdx < item.Scheme.indexes_size()) { - item.State = EState::BuildIndexes; - AllocateTxId(importInfo, itemIdx); - } else { - item.State = EState::Done; - } - } - break; - - case EState::BuildIndexes: - if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { - item.Issue = *issue; - Cancel(importInfo, itemIdx, "issues during index building"); - } else { - if (++item.NextIndexIdx < item.Scheme.indexes_size()) { - AllocateTxId(importInfo, itemIdx); - } else { - item.State = EState::Done; - } - } - break; - - default: - return SendNotificationsIfFinished(importInfo); - } - - if (AllOf(importInfo->Items, &TImportInfo::TItem::IsDone)) { - importInfo->State = EState::Done; - } - - Self->PersistImportItemState(db, importInfo, itemIdx); - Self->PersistImportState(db, importInfo); - - SendNotificationsIfFinished(importInfo); - } - -}; // TTxProgress - + if (record.GetPathCreateTxId()) { + txId = TTxId(record.GetPathCreateTxId()); + } else if (item.State == EState::Transferring) { + txId = GetActiveRestoreTxId(importInfo, itemIdx); + } + } + + if (txId == InvalidTxId) { + item.Issue = record.GetReason(); + Self->PersistImportItemState(db, importInfo, itemIdx); + + if (importInfo->State != EState::Waiting) { + return; + } + + Cancel(importInfo, itemIdx, "unhappy propose"); + Self->PersistImportState(db, importInfo); + + return SendNotificationsIfFinished(importInfo); + } + + Self->TxIdToImport[txId] = {importInfo->Id, itemIdx}; + } + + item.WaitTxId = txId; + Self->PersistImportItemState(db, importInfo, itemIdx); + + if (importInfo->State != EState::Waiting && item.State == EState::Transferring) { + CancelTransferring(importInfo, itemIdx); + return; + } + + if (item.State == EState::CreateTable) { + item.DstPathId = Self->MakeLocalId(TLocalPathId(record.GetPathId())); + Self->PersistImportItemDstPathId(db, importInfo, itemIdx); + } + + SubscribeTx(importInfo, itemIdx); + } + + void OnCreateIndexResult(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(CreateIndexResult); + const auto& record = CreateIndexResult->Get()->Record; + + LOG_D("TImport::TTxProgress: OnCreateIndexResult" + << ": txId# " << record.GetTxId() + << ", status# " << record.GetStatus()); + LOG_T("Message:\n" << record.ShortDebugString()); + + auto txId = TTxId(record.GetTxId()); + if (!Self->TxIdToImport.contains(txId)) { + LOG_E("TImport::TTxProgress: OnCreateIndexResult received unknown txId" + << ": txId# " << txId); + return; + } + + ui64 id; + ui32 itemIdx; + std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); + if (!Self->Imports.contains(id)) { + LOG_E("TImport::TTxProgress: OnCreateIndexResult received unknown id" + << ": id# " << id); + return; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(id); + NIceDb::TNiceDb db(txc.DB); + + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { + Self->TxIdToImport.erase(txId); + txId = InvalidTxId; + + if (record.GetStatus() == Ydb::StatusIds::ALREADY_EXISTS) { + if (item.State == EState::BuildIndexes) { + txId = GetActiveBuildIndexId(importInfo, itemIdx); + } + } + + if (txId == InvalidTxId) { + item.Issue = GetIssues(record); + Self->PersistImportItemState(db, importInfo, itemIdx); + + if (importInfo->State != EState::Waiting) { + return; + } + + Cancel(importInfo, itemIdx, "unhappy propose"); + Self->PersistImportState(db, importInfo); + + return SendNotificationsIfFinished(importInfo); + } + + Self->TxIdToImport[txId] = {importInfo->Id, itemIdx}; + } + + item.WaitTxId = txId; + Self->PersistImportItemState(db, importInfo, itemIdx); + + if (importInfo->State != EState::Waiting) { + CancelIndexBuilding(importInfo, itemIdx); + return; + } + + SubscribeTx(importInfo, itemIdx); + } + + void OnNotifyResult(TTransactionContext& txc, const TActorContext&) { + Y_VERIFY(NotifyResult); + const auto& record = NotifyResult->Get()->Record; + + LOG_D("TImport::TTxProgress: OnNotifyResult" + << ": txId# " << record.GetTxId()); + + const auto txId = TTxId(record.GetTxId()); + if (!Self->TxIdToImport.contains(txId)) { + LOG_E("TImport::TTxProgress: OnNotifyResult received unknown txId" + << ": txId# " << txId); + return; + } + + ui64 id; + ui32 itemIdx; + std::tie(id, itemIdx) = Self->TxIdToImport.at(txId); + if (!Self->Imports.contains(id)) { + LOG_E("TImport::TTxProgress: OnNotifyResult received unknown id" + << ": id# " << id); + return; + } + + TImportInfo::TPtr importInfo = Self->Imports.at(id); + NIceDb::TNiceDb db(txc.DB); + + Y_VERIFY(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.WaitTxId = InvalidTxId; + Self->PersistImportItemState(db, importInfo, itemIdx); + + Self->TxIdToImport.erase(txId); + + if (importInfo->State != EState::Waiting) { + return; + } + + switch (item.State) { + case EState::CreateTable: + item.State = EState::Transferring; + AllocateTxId(importInfo, itemIdx); + break; + + case EState::Transferring: + if (const auto issue = GetIssues(item.DstPathId, txId)) { + item.Issue = *issue; + Cancel(importInfo, itemIdx, "issues during restore"); + } else { + if (item.NextIndexIdx < item.Scheme.indexes_size()) { + item.State = EState::BuildIndexes; + AllocateTxId(importInfo, itemIdx); + } else { + item.State = EState::Done; + } + } + break; + + case EState::BuildIndexes: + if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { + item.Issue = *issue; + Cancel(importInfo, itemIdx, "issues during index building"); + } else { + if (++item.NextIndexIdx < item.Scheme.indexes_size()) { + AllocateTxId(importInfo, itemIdx); + } else { + item.State = EState::Done; + } + } + break; + + default: + return SendNotificationsIfFinished(importInfo); + } + + if (AllOf(importInfo->Items, &TImportInfo::TItem::IsDone)) { + importInfo->State = EState::Done; + } + + Self->PersistImportItemState(db, importInfo, itemIdx); + Self->PersistImportState(db, importInfo); + + SendNotificationsIfFinished(importInfo); + } + +}; // TTxProgress + ITransaction* TSchemeShard::CreateTxCreateImport(TEvImport::TEvCreateImportRequest::TPtr& ev) { - return new TImport::TTxCreate(this, ev); -} - + return new TImport::TTxCreate(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressImport(ui64 id) { - return new TImport::TTxProgress(this, id); -} - + return new TImport::TTxProgress(this, id); +} + ITransaction* TSchemeShard::CreateTxProgressImport(TEvPrivate::TEvImportSchemeReady::TPtr& ev) { - return new TImport::TTxProgress(this, ev); -} - + return new TImport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressImport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev) { - return new TImport::TTxProgress(this, ev); -} - + return new TImport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressImport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev) { - return new TImport::TTxProgress(this, ev); -} - + return new TImport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressImport(TEvIndexBuilder::TEvCreateResponse::TPtr& ev) { - return new TImport::TTxProgress(this, ev); -} - + return new TImport::TTxProgress(this, ev); +} + ITransaction* TSchemeShard::CreateTxProgressImport(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) { - return new TImport::TTxProgress(this, ev); -} - + return new TImport::TTxProgress(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import__forget.cpp b/ydb/core/tx/schemeshard/schemeshard_import__forget.cpp index 1a4a9fe7238..5574044683d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__forget.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__forget.cpp @@ -1,80 +1,80 @@ -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_import_helpers.h" -#include "schemeshard_import.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_import_helpers.h" +#include "schemeshard_import.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/generic/ptr.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TImport::TTxForget: public TSchemeShard::TXxport::TTxBase { - TEvImport::TEvForgetImportRequest::TPtr Request; - - explicit TTxForget(TSelf *self, TEvImport::TEvForgetImportRequest::TPtr& ev) - : TXxport::TTxBase(self) - , Request(ev) - { - } - - TTxType GetTxType() const override { - return TXTYPE_FORGET_IMPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext&) override { - const auto& request = Request->Get()->Record; - - auto respond = [this, &request](Ydb::StatusIds::StatusCode status, const TString& issue = {}) -> bool { - auto response = MakeHolder<TEvImport::TEvForgetImportResponse>(request.GetTxId()); - auto& proto = *response->Record.MutableResponse(); - - proto.SetStatus(status); - if (issue) { - AddIssue(proto, issue); - } - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - }; - - auto it = Self->Imports.find(request.GetRequest().GetId()); - if (it == Self->Imports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { - return respond(Ydb::StatusIds::NOT_FOUND); - } - - TImportInfo::TPtr importInfo = it->second; - NIceDb::TNiceDb db(txc.DB); - - switch (importInfo->State) { - case TImportInfo::EState::Done: - case TImportInfo::EState::Cancelled: - Self->ImportsByUid.erase(importInfo->Uid); - Self->Imports.erase(importInfo->Id); - Self->PersistRemoveImport(db, importInfo); - return respond(Ydb::StatusIds::SUCCESS); - - case TImportInfo::EState::Waiting: - case TImportInfo::EState::Cancellation: - return respond(Ydb::StatusIds::PRECONDITION_FAILED, "Import operation is in progress"); - - default: - return respond(Ydb::StatusIds::UNDETERMINED); - } - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxForget - + TEvImport::TEvForgetImportRequest::TPtr Request; + + explicit TTxForget(TSelf *self, TEvImport::TEvForgetImportRequest::TPtr& ev) + : TXxport::TTxBase(self) + , Request(ev) + { + } + + TTxType GetTxType() const override { + return TXTYPE_FORGET_IMPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext&) override { + const auto& request = Request->Get()->Record; + + auto respond = [this, &request](Ydb::StatusIds::StatusCode status, const TString& issue = {}) -> bool { + auto response = MakeHolder<TEvImport::TEvForgetImportResponse>(request.GetTxId()); + auto& proto = *response->Record.MutableResponse(); + + proto.SetStatus(status); + if (issue) { + AddIssue(proto, issue); + } + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + }; + + auto it = Self->Imports.find(request.GetRequest().GetId()); + if (it == Self->Imports.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { + return respond(Ydb::StatusIds::NOT_FOUND); + } + + TImportInfo::TPtr importInfo = it->second; + NIceDb::TNiceDb db(txc.DB); + + switch (importInfo->State) { + case TImportInfo::EState::Done: + case TImportInfo::EState::Cancelled: + Self->ImportsByUid.erase(importInfo->Uid); + Self->Imports.erase(importInfo->Id); + Self->PersistRemoveImport(db, importInfo); + return respond(Ydb::StatusIds::SUCCESS); + + case TImportInfo::EState::Waiting: + case TImportInfo::EState::Cancellation: + return respond(Ydb::StatusIds::PRECONDITION_FAILED, "Import operation is in progress"); + + default: + return respond(Ydb::StatusIds::UNDETERMINED); + } + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxForget + ITransaction* TSchemeShard::CreateTxForgetImport(TEvImport::TEvForgetImportRequest::TPtr& ev) { - return new TImport::TTxForget(this, ev); -} - + return new TImport::TTxForget(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import__get.cpp b/ydb/core/tx/schemeshard/schemeshard_import__get.cpp index aedfbdc3110..dcc8cb9c8f2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__get.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__get.cpp @@ -1,31 +1,31 @@ -#include "schemeshard_xxport__get.h" -#include "schemeshard_import.h" - -namespace NKikimr { +#include "schemeshard_xxport__get.h" +#include "schemeshard_import.h" + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TImport::TTxGet: public TSchemeShard::TXxport::TTxGet< - TImportInfo, - TEvImport::TEvGetImportRequest, - TEvImport::TEvGetImportResponse -> { - using TTxGetBase::TTxGetBase; - - TTxType GetTxType() const override { - return TXTYPE_GET_IMPORT; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - return DoExecuteImpl(Self->Imports, txc, ctx); - } - -}; // TTxGet - + TImportInfo, + TEvImport::TEvGetImportRequest, + TEvImport::TEvGetImportResponse +> { + using TTxGetBase::TTxGetBase; + + TTxType GetTxType() const override { + return TXTYPE_GET_IMPORT; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + return DoExecuteImpl(Self->Imports, txc, ctx); + } + +}; // TTxGet + ITransaction* TSchemeShard::CreateTxGetImport(TEvImport::TEvGetImportRequest::TPtr& ev) { - return new TImport::TTxGet(this, ev); -} - + return new TImport::TTxGet(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import__list.cpp b/ydb/core/tx/schemeshard/schemeshard_import__list.cpp index c45ee198c98..c2841669c8a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__list.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__list.cpp @@ -1,41 +1,41 @@ -#include "schemeshard_xxport__list.h" -#include "schemeshard_import.h" - -namespace NKikimr { +#include "schemeshard_xxport__list.h" +#include "schemeshard_import.h" + +namespace NKikimr { namespace NSchemeShard { - -using namespace NTabletFlatExecutor; - + +using namespace NTabletFlatExecutor; + struct TSchemeShard::TImport::TTxList: public TSchemeShard::TXxport::TTxList< - TImportInfo, - TEvImport::TEvListImportsRequest, - TEvImport::TEvListImportsResponse, + TImportInfo, + TEvImport::TEvListImportsRequest, + TEvImport::TEvListImportsResponse, TSchemeShard::TImport::TTxList -> { - using TTxListBase::TTxListBase; - - TTxType GetTxType() const override { - return TXTYPE_LIST_IMPORTS; - } - - static bool TryParseKind(const TString& str, TImportInfo::EKind& parsed) { - if (str == "import/s3") { - parsed = TImportInfo::EKind::S3; - return true; - } - - return false; - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - return DoExecuteImpl(Self->Imports, txc, ctx); - } - -}; // TTxList - +> { + using TTxListBase::TTxListBase; + + TTxType GetTxType() const override { + return TXTYPE_LIST_IMPORTS; + } + + static bool TryParseKind(const TString& str, TImportInfo::EKind& parsed) { + if (str == "import/s3") { + parsed = TImportInfo::EKind::S3; + return true; + } + + return false; + } + + bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { + return DoExecuteImpl(Self->Imports, txc, ctx); + } + +}; // TTxList + ITransaction* TSchemeShard::CreateTxListImports(TEvImport::TEvListImportsRequest::TPtr& ev) { - return new TImport::TTxList(this, ev); -} - + return new TImport::TTxList(this, ev); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 13fce9e7f9e..5d510760fd4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -1,198 +1,198 @@ -#include "schemeshard_import_flow_proposals.h" -#include "schemeshard_path_describer.h" - +#include "schemeshard_import_flow_proposals.h" +#include "schemeshard_path_describer.h" + #include <ydb/core/base/path.h> #include <ydb/core/ydb_convert/table_description.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx, - TString& error -) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx, + TString& error +) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - auto& record = propose->Record; - - if (importInfo->UserSID) { - record.SetOwner(*importInfo->UserSID); - } - + auto& record = propose->Record; + + if (importInfo->UserSID) { + record.SetOwner(*importInfo->UserSID); + } + auto& modifyScheme = *record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateTable); - modifyScheme.SetInternal(true); - - const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); - - std::pair<TString, TString> wdAndPath; - if (!TrySplitPathByDb(item.DstPathName, domainPath.PathString(), wdAndPath, error)) { - return nullptr; - } - - modifyScheme.SetWorkingDir(wdAndPath.first); - - auto& tableDesc = *modifyScheme.MutableCreateTable(); - tableDesc.SetName(wdAndPath.second); - - Ydb::StatusIds::StatusCode status; - if (!FillTableDescription(modifyScheme, item.Scheme, status, error)) { - return nullptr; - } - - return propose; -} - + modifyScheme.SetInternal(true); + + const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); + + std::pair<TString, TString> wdAndPath; + if (!TrySplitPathByDb(item.DstPathName, domainPath.PathString(), wdAndPath, error)) { + return nullptr; + } + + modifyScheme.SetWorkingDir(wdAndPath.first); + + auto& tableDesc = *modifyScheme.MutableCreateTable(); + tableDesc.SetName(wdAndPath.second); + + Ydb::StatusIds::StatusCode status; + if (!FillTableDescription(modifyScheme, item.Scheme, status, error)) { + return nullptr; + } + + return propose; +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx -) { - TString unused; - return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); -} - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx +) { + TString unused; + return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); +} + static NKikimrSchemeOp::TTableDescription GetTableDescription(TSchemeShard* ss, const TPathId& pathId) { - auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId); - auto record = desc->GetRecord(); - - Y_VERIFY(record.HasPathDescription()); - Y_VERIFY(record.GetPathDescription().HasTable()); - - return record.GetPathDescription().GetTable(); -} - + auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId); + auto record = desc->GetRecord(); + + Y_VERIFY(record.HasPathDescription()); + Y_VERIFY(record.GetPathDescription().HasTable()); + + return record.GetPathDescription().GetTable(); +} + static NKikimrSchemeOp::TTableDescription RebuildTableDescription( const NKikimrSchemeOp::TTableDescription& src, - const Ydb::Table::CreateTableRequest& scheme -) { + const Ydb::Table::CreateTableRequest& scheme +) { NKikimrSchemeOp::TTableDescription tableDesc; - tableDesc.MutableKeyColumnNames()->CopyFrom(src.GetKeyColumnNames()); - - THashMap<TString, ui32> columnNameToIdx; - for (ui32 i = 0; i < src.ColumnsSize(); ++i) { - Y_VERIFY(columnNameToIdx.emplace(src.GetColumns(i).GetName(), i).second); - } - - for (const auto& column : scheme.columns()) { - auto it = columnNameToIdx.find(column.name()); - Y_VERIFY(it != columnNameToIdx.end()); - - Y_VERIFY(it->second < src.ColumnsSize()); - tableDesc.MutableColumns()->Add()->CopyFrom(src.GetColumns(it->second)); - } - - return tableDesc; -} - + tableDesc.MutableKeyColumnNames()->CopyFrom(src.GetKeyColumnNames()); + + THashMap<TString, ui32> columnNameToIdx; + for (ui32 i = 0; i < src.ColumnsSize(); ++i) { + Y_VERIFY(columnNameToIdx.emplace(src.GetColumns(i).GetName(), i).second); + } + + for (const auto& column : scheme.columns()) { + auto it = columnNameToIdx.find(column.name()); + Y_VERIFY(it != columnNameToIdx.end()); + + Y_VERIFY(it->second < src.ColumnsSize()); + tableDesc.MutableColumns()->Add()->CopyFrom(src.GetColumns(it->second)); + } + + return tableDesc; +} + THolder<TEvSchemeShard::TEvModifySchemeTransaction> RestorePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx -) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx +) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); - + auto& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpRestore); - modifyScheme.SetInternal(true); - - const TPath dstPath = TPath::Init(item.DstPathId, ss); - Y_VERIFY(dstPath.IsResolved()); - - modifyScheme.SetWorkingDir(dstPath.Parent().PathString()); - - auto& task = *modifyScheme.MutableRestore(); - task.SetTableName(dstPath.LeafName()); - *task.MutableTableDescription() = RebuildTableDescription(GetTableDescription(ss, item.DstPathId), item.Scheme); - - switch (importInfo->Kind) { - case TImportInfo::EKind::S3: - { - task.SetNumberOfRetries(importInfo->Settings.number_of_retries()); - auto& restoreSettings = *task.MutableS3Settings(); - restoreSettings.SetEndpoint(importInfo->Settings.endpoint()); - restoreSettings.SetBucket(importInfo->Settings.bucket()); - restoreSettings.SetAccessKey(importInfo->Settings.access_key()); - restoreSettings.SetSecretKey(importInfo->Settings.secret_key()); - restoreSettings.SetObjectKeyPattern(importInfo->Settings.items(itemIdx).source_prefix()); - - switch (importInfo->Settings.scheme()) { - case Ydb::Import::ImportFromS3Settings::HTTP: + modifyScheme.SetInternal(true); + + const TPath dstPath = TPath::Init(item.DstPathId, ss); + Y_VERIFY(dstPath.IsResolved()); + + modifyScheme.SetWorkingDir(dstPath.Parent().PathString()); + + auto& task = *modifyScheme.MutableRestore(); + task.SetTableName(dstPath.LeafName()); + *task.MutableTableDescription() = RebuildTableDescription(GetTableDescription(ss, item.DstPathId), item.Scheme); + + switch (importInfo->Kind) { + case TImportInfo::EKind::S3: + { + task.SetNumberOfRetries(importInfo->Settings.number_of_retries()); + auto& restoreSettings = *task.MutableS3Settings(); + restoreSettings.SetEndpoint(importInfo->Settings.endpoint()); + restoreSettings.SetBucket(importInfo->Settings.bucket()); + restoreSettings.SetAccessKey(importInfo->Settings.access_key()); + restoreSettings.SetSecretKey(importInfo->Settings.secret_key()); + restoreSettings.SetObjectKeyPattern(importInfo->Settings.items(itemIdx).source_prefix()); + + switch (importInfo->Settings.scheme()) { + case Ydb::Import::ImportFromS3Settings::HTTP: restoreSettings.SetScheme(NKikimrSchemeOp::TS3Settings::HTTP); - break; - case Ydb::Import::ImportFromS3Settings::HTTPS: + break; + case Ydb::Import::ImportFromS3Settings::HTTPS: restoreSettings.SetScheme(NKikimrSchemeOp::TS3Settings::HTTPS); - break; - default: - Y_FAIL("Unknown scheme"); - } - } - break; - } - - return propose; -} - + break; + default: + Y_FAIL("Unknown scheme"); + } + } + break; + } + + return propose; +} + THolder<TEvSchemeShard::TEvCancelTx> CancelRestorePropose( - TImportInfo::TPtr importInfo, - TTxId restoreTxId -) { + TImportInfo::TPtr importInfo, + TTxId restoreTxId +) { auto propose = MakeHolder<TEvSchemeShard::TEvCancelTx>(); - - auto& record = propose->Record; - record.SetTxId(importInfo->Id); - record.SetTargetTxId(ui64(restoreTxId)); - - return propose; -} - -THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose( + + auto& record = propose->Record; + record.SetTxId(importInfo->Id); + record.SetTargetTxId(ui64(restoreTxId)); + + return propose; +} + +THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx, - const TString& uid -) { - Y_VERIFY(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - NKikimrIndexBuilder::TIndexBuildSettings settings; - - const TPath dstPath = TPath::Init(item.DstPathId, ss); - settings.set_source_path(dstPath.PathString()); - - Y_VERIFY(item.NextIndexIdx < item.Scheme.indexes_size()); - settings.mutable_index()->CopyFrom(item.Scheme.indexes(item.NextIndexIdx)); - - if (settings.mutable_index()->type_case() == Ydb::Table::TableIndex::TypeCase::TYPE_NOT_SET) { - settings.mutable_index()->mutable_global_index(); - } - - const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); - auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings)); - (*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid; - - return propose; -} - -THolder<TEvIndexBuilder::TEvCancelRequest> CancelIndexBuildPropose( + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx, + const TString& uid +) { + Y_VERIFY(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + NKikimrIndexBuilder::TIndexBuildSettings settings; + + const TPath dstPath = TPath::Init(item.DstPathId, ss); + settings.set_source_path(dstPath.PathString()); + + Y_VERIFY(item.NextIndexIdx < item.Scheme.indexes_size()); + settings.mutable_index()->CopyFrom(item.Scheme.indexes(item.NextIndexIdx)); + + if (settings.mutable_index()->type_case() == Ydb::Table::TableIndex::TypeCase::TYPE_NOT_SET) { + settings.mutable_index()->mutable_global_index(); + } + + const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); + auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings)); + (*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid; + + return propose; +} + +THolder<TEvIndexBuilder::TEvCancelRequest> CancelIndexBuildPropose( TSchemeShard* ss, - TImportInfo::TPtr importInfo, - TTxId indexBuildId -) { - const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); - return MakeHolder<TEvIndexBuilder::TEvCancelRequest>(ui64(indexBuildId), domainPath.PathString(), ui64(indexBuildId)); -} - + TImportInfo::TPtr importInfo, + TTxId indexBuildId +) { + const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss); + return MakeHolder<TEvIndexBuilder::TEvCancelRequest>(ui64(indexBuildId), domainPath.PathString(), ui64(indexBuildId)); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h index b466a5714cd..a99221db35b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h @@ -1,50 +1,50 @@ -#pragma once - -#include "schemeshard_impl.h" - -namespace NKikimr { +#pragma once + +#include "schemeshard_impl.h" + +namespace NKikimr { namespace NSchemeShard { - + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx, - TString& error -); - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx, + TString& error +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx -); - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx +); + THolder<TEvSchemeShard::TEvModifySchemeTransaction> RestorePropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx -); - + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx +); + THolder<TEvSchemeShard::TEvCancelTx> CancelRestorePropose( - TImportInfo::TPtr importInfo, - TTxId restoreTxId -); - -THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose( + TImportInfo::TPtr importInfo, + TTxId restoreTxId +); + +THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose( TSchemeShard* ss, - TTxId txId, - TImportInfo::TPtr importInfo, - ui32 itemIdx, - const TString& uid -); - -THolder<TEvIndexBuilder::TEvCancelRequest> CancelIndexBuildPropose( + TTxId txId, + TImportInfo::TPtr importInfo, + ui32 itemIdx, + const TString& uid +); + +THolder<TEvIndexBuilder::TEvCancelRequest> CancelIndexBuildPropose( TSchemeShard* ss, - TImportInfo::TPtr importInfo, - TTxId indexBuildId -); - + TImportInfo::TPtr importInfo, + TTxId indexBuildId +); + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_helpers.h b/ydb/core/tx/schemeshard/schemeshard_import_helpers.h index cb60634e20d..3990290353f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_helpers.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_helpers.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/protos/issue_severity.pb.h> - -#if defined LOG_T || \ - defined LOG_D || \ - defined LOG_I || \ - defined LOG_N || \ - defined LOG_W || \ - defined LOG_W -#error log macro redefinition -#endif - -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) - -template <typename T> -void AddIssue(T& response, const TString& message, NYql::TSeverityIds::ESeverityId severity = NYql::TSeverityIds::S_ERROR) { - auto& issue = *response.AddIssues(); - issue.set_severity(severity); - issue.set_message(message); -} + +#if defined LOG_T || \ + defined LOG_D || \ + defined LOG_I || \ + defined LOG_N || \ + defined LOG_W || \ + defined LOG_W +#error log macro redefinition +#endif + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::IMPORT, stream) + +template <typename T> +void AddIssue(T& response, const TString& message, NYql::TSeverityIds::ESeverityId severity = NYql::TSeverityIds::S_ERROR) { + auto& issue = *response.AddIssues(); + issue.set_severity(severity); + issue.set_message(message); +} diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 27060076f41..e2fc78da5dc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -1,204 +1,204 @@ -#include "schemeshard_import_scheme_getter.h" -#include "schemeshard_import_helpers.h" -#include "schemeshard_private.h" - +#include "schemeshard_import_scheme_getter.h" +#include "schemeshard_import_helpers.h" +#include "schemeshard_private.h" + #include <ydb/core/wrappers/s3_wrapper.h> #include <ydb/public/api/protos/ydb_import.pb.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> - -#include <util/string/subst.h> - -namespace NKikimr { + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> + +#include <util/string/subst.h> + +namespace NKikimr { namespace NSchemeShard { - -using namespace NWrappers; - -using namespace Aws::Auth; -using namespace Aws::Client; -using namespace Aws::S3; -using namespace Aws; - + +using namespace NWrappers; + +using namespace Aws::Auth; +using namespace Aws::Client; +using namespace Aws::S3; +using namespace Aws; + class TSchemeGetter: public TActorBootstrapped<TSchemeGetter>, private TS3User { - static ClientConfiguration ConfigFromSettings(const Ydb::Import::ImportFromS3Settings& settings) { - ClientConfiguration config; - - config.endpointOverride = settings.endpoint(); - config.verifySSL = false; - config.connectTimeoutMs = 10000; - config.maxConnections = 5; - - switch (settings.scheme()) { - case Ydb::Import::ImportFromS3Settings::HTTP: - config.scheme = Http::Scheme::HTTP; - break; - case Ydb::Import::ImportFromS3Settings::HTTPS: - config.scheme = Http::Scheme::HTTPS; - break; - default: - Y_FAIL("Unknown scheme"); - } - - return config; - } - - static AWSCredentials CredentialsFromSettings(const Ydb::Import::ImportFromS3Settings& settings) { - return AWSCredentials(settings.access_key(), settings.secret_key()); - } - - static TString SchemeKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) { - Y_VERIFY(itemIdx < (ui32)settings.items_size()); - return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/scheme.pb"; - } - - void HeadObject(const TString& key) { - auto request = Model::HeadObjectRequest() - .WithBucket(ImportInfo->Settings.bucket()) - .WithKey(key); - - Send(Client, new TEvS3Wrapper::TEvHeadObjectRequest(request)); - } - - void Handle(TEvS3Wrapper::TEvHeadObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - LOG_D("Handle TEvS3Wrapper::TEvHeadObjectResponse" - << ": self# " << SelfId() - << ", result# " << result); - - if (!CheckResult(result, "HeadObject")) { - return; - } - - const auto contentLength = result.GetResult().GetContentLength(); - GetObject(SchemeKey, std::make_pair(0, contentLength - 1)); - } - - void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { - auto request = Model::GetObjectRequest() - .WithBucket(ImportInfo->Settings.bucket()) - .WithKey(key) - .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); - - Send(Client, new TEvS3Wrapper::TEvGetObjectRequest(request)); - } - - void Handle(TEvS3Wrapper::TEvGetObjectResponse::TPtr& ev) { - const auto& msg = *ev->Get(); - const auto& result = msg.Result; - - LOG_D("Handle TEvS3Wrapper::TEvGetObjectResponse" - << ": self# " << SelfId() - << ", result# " << result); - - if (!CheckResult(result, "GetObject")) { - return; - } - - Y_VERIFY(ItemIdx < ImportInfo->Items.size()); - auto& item = ImportInfo->Items.at(ItemIdx); - - LOG_T("Trying to parse" - << ": self# " << SelfId() - << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); - - if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &item.Scheme)) { - return Reply(false, "Cannot parse scheme"); - } - - Reply(); - } - - template <typename TResult> - bool CheckResult(const TResult& result, const TStringBuf marker) { - if (result.IsSuccess()) { - return true; - } - - LOG_E("Error at '" << marker << "'" - << ": self# " << SelfId() - << ", error# " << result); - MaybeRetry(result.GetError().GetMessage().c_str()); - - return false; - } - - void MaybeRetry(const TString& error) { - if (Attempt++ < Retries) { - Schedule(TDuration::Minutes(1), new TEvents::TEvWakeup()); - } else { - Reply(false, error); - } - } - - void Reply(bool success = true, const TString& error = TString()) { - LOG_I("Reply" - << ": self# " << SelfId() - << ", success# " << success - << ", error# " << error); - - Send(ReplyTo, new TEvPrivate::TEvImportSchemeReady(ImportInfo->Id, ItemIdx, success, error)); - PassAway(); - } - - void PassAway() override { - Send(Client, new TEvents::TEvPoisonPill()); - TActor::PassAway(); - } - -public: - explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) - : ReplyTo(replyTo) - , ImportInfo(importInfo) - , ItemIdx(itemIdx) - , Config(ConfigFromSettings(importInfo->Settings)) - , Credentials(CredentialsFromSettings(importInfo->Settings)) - , SchemeKey(SchemeKeyFromSettings(importInfo->Settings, itemIdx)) - , Retries(importInfo->Settings.number_of_retries()) - { - } - - void Bootstrap() { - if (Client) { - Send(Client, new TEvents::TEvPoisonPill()); - } - + static ClientConfiguration ConfigFromSettings(const Ydb::Import::ImportFromS3Settings& settings) { + ClientConfiguration config; + + config.endpointOverride = settings.endpoint(); + config.verifySSL = false; + config.connectTimeoutMs = 10000; + config.maxConnections = 5; + + switch (settings.scheme()) { + case Ydb::Import::ImportFromS3Settings::HTTP: + config.scheme = Http::Scheme::HTTP; + break; + case Ydb::Import::ImportFromS3Settings::HTTPS: + config.scheme = Http::Scheme::HTTPS; + break; + default: + Y_FAIL("Unknown scheme"); + } + + return config; + } + + static AWSCredentials CredentialsFromSettings(const Ydb::Import::ImportFromS3Settings& settings) { + return AWSCredentials(settings.access_key(), settings.secret_key()); + } + + static TString SchemeKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) { + Y_VERIFY(itemIdx < (ui32)settings.items_size()); + return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/scheme.pb"; + } + + void HeadObject(const TString& key) { + auto request = Model::HeadObjectRequest() + .WithBucket(ImportInfo->Settings.bucket()) + .WithKey(key); + + Send(Client, new TEvS3Wrapper::TEvHeadObjectRequest(request)); + } + + void Handle(TEvS3Wrapper::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("Handle TEvS3Wrapper::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "HeadObject")) { + return; + } + + const auto contentLength = result.GetResult().GetContentLength(); + GetObject(SchemeKey, std::make_pair(0, contentLength - 1)); + } + + void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { + auto request = Model::GetObjectRequest() + .WithBucket(ImportInfo->Settings.bucket()) + .WithKey(key) + .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); + + Send(Client, new TEvS3Wrapper::TEvGetObjectRequest(request)); + } + + void Handle(TEvS3Wrapper::TEvGetObjectResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + const auto& result = msg.Result; + + LOG_D("Handle TEvS3Wrapper::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "GetObject")) { + return; + } + + Y_VERIFY(ItemIdx < ImportInfo->Items.size()); + auto& item = ImportInfo->Items.at(ItemIdx); + + LOG_T("Trying to parse" + << ": self# " << SelfId() + << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); + + if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &item.Scheme)) { + return Reply(false, "Cannot parse scheme"); + } + + Reply(); + } + + template <typename TResult> + bool CheckResult(const TResult& result, const TStringBuf marker) { + if (result.IsSuccess()) { + return true; + } + + LOG_E("Error at '" << marker << "'" + << ": self# " << SelfId() + << ", error# " << result); + MaybeRetry(result.GetError().GetMessage().c_str()); + + return false; + } + + void MaybeRetry(const TString& error) { + if (Attempt++ < Retries) { + Schedule(TDuration::Minutes(1), new TEvents::TEvWakeup()); + } else { + Reply(false, error); + } + } + + void Reply(bool success = true, const TString& error = TString()) { + LOG_I("Reply" + << ": self# " << SelfId() + << ", success# " << success + << ", error# " << error); + + Send(ReplyTo, new TEvPrivate::TEvImportSchemeReady(ImportInfo->Id, ItemIdx, success, error)); + PassAway(); + } + + void PassAway() override { + Send(Client, new TEvents::TEvPoisonPill()); + TActor::PassAway(); + } + +public: + explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) + : ReplyTo(replyTo) + , ImportInfo(importInfo) + , ItemIdx(itemIdx) + , Config(ConfigFromSettings(importInfo->Settings)) + , Credentials(CredentialsFromSettings(importInfo->Settings)) + , SchemeKey(SchemeKeyFromSettings(importInfo->Settings, itemIdx)) + , Retries(importInfo->Settings.number_of_retries()) + { + } + + void Bootstrap() { + if (Client) { + Send(Client, new TEvents::TEvPoisonPill()); + } + Client = RegisterWithSameMailbox(CreateS3Wrapper(Credentials, Config)); - - HeadObject(SchemeKey); - Become(&TThis::StateWork); - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvS3Wrapper::TEvHeadObjectResponse, Handle); - hFunc(TEvS3Wrapper::TEvGetObjectResponse, Handle); - - cFunc(TEvents::TEvWakeup::EventType, Bootstrap); - cFunc(TEvents::TEvPoisonPill::EventType, PassAway); - } - } - -private: - const TActorId ReplyTo; - TImportInfo::TPtr ImportInfo; - const ui32 ItemIdx; - - const ClientConfiguration Config; - const AWSCredentials Credentials; - const TString SchemeKey; - - const ui32 Retries; - ui32 Attempt = 0; - - TActorId Client; - -}; // TSchemeGetter - -IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) { - return new TSchemeGetter(replyTo, importInfo, itemIdx); -} - + + HeadObject(SchemeKey); + Become(&TThis::StateWork); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvS3Wrapper::TEvHeadObjectResponse, Handle); + hFunc(TEvS3Wrapper::TEvGetObjectResponse, Handle); + + cFunc(TEvents::TEvWakeup::EventType, Bootstrap); + cFunc(TEvents::TEvPoisonPill::EventType, PassAway); + } + } + +private: + const TActorId ReplyTo; + TImportInfo::TPtr ImportInfo; + const ui32 ItemIdx; + + const ClientConfiguration Config; + const AWSCredentials Credentials; + const TString SchemeKey; + + const ui32 Retries; + ui32 Attempt = 0; + + TActorId Client; + +}; // TSchemeGetter + +IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) { + return new TSchemeGetter(replyTo, importInfo, itemIdx); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.h b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.h index 5899344832d..24b67c7c659 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.h @@ -1,12 +1,12 @@ -#pragma once - -#include "defs.h" -#include "schemeshard_info_types.h" - -namespace NKikimr { +#pragma once + +#include "defs.h" +#include "schemeshard_info_types.h" + +namespace NKikimr { namespace NSchemeShard { - -IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx); - + +IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx); + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter_fallback.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter_fallback.cpp index 6d0e34e74b1..eefb0e9e5a0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter_fallback.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter_fallback.cpp @@ -1,36 +1,36 @@ -#include "schemeshard_import_scheme_getter.h" -#include "schemeshard_private.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> - -namespace NKikimr { +#include "schemeshard_import_scheme_getter.h" +#include "schemeshard_private.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> + +namespace NKikimr { namespace NSchemeShard { - -class TSchemeGetterFallback: public TActorBootstrapped<TSchemeGetterFallback> { -public: - explicit TSchemeGetterFallback(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) - : ReplyTo(replyTo) - , ImportInfo(importInfo) - , ItemIdx(itemIdx) - { - } - - void Bootstrap() { - Send(ReplyTo, new TEvPrivate::TEvImportSchemeReady(ImportInfo->Id, ItemIdx, false, "Imports from S3 are disabled")); - PassAway(); - } - -private: - const TActorId ReplyTo; - TImportInfo::TPtr ImportInfo; - const ui32 ItemIdx; - -}; // TSchemeGetterFallback - -IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) { - return new TSchemeGetterFallback(replyTo, importInfo, itemIdx); -} - + +class TSchemeGetterFallback: public TActorBootstrapped<TSchemeGetterFallback> { +public: + explicit TSchemeGetterFallback(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) + : ReplyTo(replyTo) + , ImportInfo(importInfo) + , ItemIdx(itemIdx) + { + } + + void Bootstrap() { + Send(ReplyTo, new TEvPrivate::TEvImportSchemeReady(ImportInfo->Id, ItemIdx, false, "Imports from S3 are disabled")); + PassAway(); + } + +private: + const TActorId ReplyTo; + TImportInfo::TPtr ImportInfo; + const ui32 ItemIdx; + +}; // TSchemeGetterFallback + +IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) { + return new TSchemeGetterFallback(replyTo, importInfo, itemIdx); +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 7f06ef0d3a8..fcb564b6065 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -12,8 +12,8 @@ #include <ydb/library/yql/minikql/mkql_type_ops.h> -#include <util/generic/algorithm.h> - +#include <util/generic/algorithm.h> + namespace NKikimr { namespace NSchemeShard { @@ -21,8 +21,8 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( TPtr source, NKikimrSchemeOp::TTableDescription& op, const NScheme::TTypeRegistry& typeRegistry, - const TSchemeLimits& limits, const TSubDomainInfo& subDomain, - TString& errStr, const THashSet<TString>& localSequences) + const TSchemeLimits& limits, const TSubDomainInfo& subDomain, + TString& errStr, const THashSet<TString>& localSequences) { TAlterDataPtr alterData = new TTableInfo::TAlterTableInfo(); alterData->TableDescriptionFull = NKikimrSchemeOp::TTableDescription(); @@ -53,14 +53,14 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( for (auto& col : *op.MutableColumns()) { TString colName = col.GetName(); - - if (colName.size() > limits.MaxTableColumnNameLength) { - errStr = TStringBuilder() - << "Column name too long '" << colName << "'. " - << "Limit: " << limits.MaxTableColumnNameLength; - return nullptr; - } - + + if (colName.size() > limits.MaxTableColumnNameLength) { + errStr = TStringBuilder() + << "Column name too long '" << colName << "'. " + << "Limit: " << limits.MaxTableColumnNameLength; + return nullptr; + } + if (!IsValidColumnName(colName)) { errStr = Sprintf("Invalid name for column '%s'", colName.data()); return nullptr; @@ -183,42 +183,42 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( errStr = Sprintf("Can't drop key column: '%s'", colName.data()); return nullptr; } - if (alterData->Columns[colId].DeleteVersion == alterData->AlterVersion) { - errStr = Sprintf("Duplicate drop column: %s", colName.data()); - return nullptr; - } - if (source->TTLSettings().HasEnabled() && source->TTLSettings().GetEnabled().GetColumnName() == colName) { - errStr = Sprintf("Can't drop TTL column: '%s', disable TTL first ", colName.data()); - return nullptr; - } - + if (alterData->Columns[colId].DeleteVersion == alterData->AlterVersion) { + errStr = Sprintf("Duplicate drop column: %s", colName.data()); + return nullptr; + } + if (source->TTLSettings().HasEnabled() && source->TTLSettings().GetEnabled().GetColumnName() == colName) { + errStr = Sprintf("Can't drop TTL column: '%s', disable TTL first ", colName.data()); + return nullptr; + } + alterData->Columns[colId] = source->Columns[colId]; alterData->Columns[colId].DeleteVersion = alterData->AlterVersion; } } - if ((colName2Id.size() - op.DropColumnsSize()) > limits.MaxTableColumns) { - errStr = TStringBuilder() - << "Too many columns" - << ": current: " << (source ? source->Columns.size() : 0) - << ", new: " << (source ? colName2Id.size() - source->Columns.size() : op.ColumnsSize()) - << ", dropping: " << op.DropColumnsSize() - << ". Limit: " << limits.MaxTableColumns; - return nullptr; - } - - if (op.HasTTLSettings()) { - const auto& ttl = op.GetTTLSettings(); - - if (!ValidateTtlSettings(ttl, source ? source->Columns : THashMap<ui32, TColumn>(), alterData->Columns, colName2Id, subDomain, errStr)) { + if ((colName2Id.size() - op.DropColumnsSize()) > limits.MaxTableColumns) { + errStr = TStringBuilder() + << "Too many columns" + << ": current: " << (source ? source->Columns.size() : 0) + << ", new: " << (source ? colName2Id.size() - source->Columns.size() : op.ColumnsSize()) + << ", dropping: " << op.DropColumnsSize() + << ". Limit: " << limits.MaxTableColumns; + return nullptr; + } + + if (op.HasTTLSettings()) { + const auto& ttl = op.GetTTLSettings(); + + if (!ValidateTtlSettings(ttl, source ? source->Columns : THashMap<ui32, TColumn>(), alterData->Columns, colName2Id, subDomain, errStr)) { return nullptr; - } - - alterData->TableDescriptionFull->MutableTTLSettings()->CopyFrom(ttl); - } - - alterData->IsBackup = op.GetIsBackup(); - + } + + alterData->TableDescriptionFull->MutableTTLSettings()->CopyFrom(ttl); + } + + alterData->IsBackup = op.GetIsBackup(); + if (source && op.KeyColumnNamesSize() == 0) return alterData; @@ -249,15 +249,15 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( ++keyOrder; } - if (keyColIds.size() > limits.MaxTableKeyColumns) { - errStr = TStringBuilder() - << "Too many key columns" - << ": current: " << (source ? source->KeyColumnIds.size() : 0) - << ", new: " << keyColIds.size() - << ". Limit: " << limits.MaxTableKeyColumns; - return nullptr; - } - + if (keyColIds.size() > limits.MaxTableKeyColumns) { + errStr = TStringBuilder() + << "Too many key columns" + << ": current: " << (source ? source->KeyColumnIds.size() : 0) + << ", new: " << keyColIds.size() + << ". Limit: " << limits.MaxTableKeyColumns; + return nullptr; + } + if (source) { // key columns reorder or deletion is not supported const TVector<ui32>& oldColIds = source->KeyColumnIds; @@ -292,7 +292,7 @@ TVector<ui32> TTableInfo::FillDescription(TPathElement::TPtr pathInfo) { if (!TableDescription.HasPathId()) { TableDescription.SetName(pathInfo->Name); TableDescription.SetId_Deprecated(pathInfo->PathId.LocalPathId); - PathIdFromPathId(pathInfo->PathId, TableDescription.MutablePathId()); + PathIdFromPathId(pathInfo->PathId, TableDescription.MutablePathId()); for (auto& c : Columns) { const TColumn& column = c.second; @@ -1191,11 +1191,11 @@ void TTableInfo::FinishAlter() { partitionConfig.ClearShadowData(); } - // Apply TTL params - if (AlterData->TableDescriptionFull.Defined() && AlterData->TableDescriptionFull->HasTTLSettings()) { - MutableTTLSettings().Swap(AlterData->TableDescriptionFull->MutableTTLSettings()); - } - + // Apply TTL params + if (AlterData->TableDescriptionFull.Defined() && AlterData->TableDescriptionFull->HasTTLSettings()) { + MutableTTLSettings().Swap(AlterData->TableDescriptionFull->MutableTTLSettings()); + } + // Force FillDescription to regenerate TableDescription TableDescription.ClearId_Deprecated(); TableDescription.ClearPathId(); @@ -1274,13 +1274,13 @@ void TTableInfo::SetPartitioning(TVector<TTableShardInfo>&& newPartitioning) { Stats.Aggregated = newAggregatedStats; Partitions.swap(newPartitioning); PreSerializedPathDescription.clear(); - - CondEraseSchedule.clear(); - InFlightCondErase.clear(); + + CondEraseSchedule.clear(); + InFlightCondErase.clear(); Shard2PartitionIdx.clear(); for (ui32 i = 0; i < Partitions.size(); ++i) { Shard2PartitionIdx[Partitions[i].ShardIdx] = i; - CondEraseSchedule.push(Partitions.begin() + i); + CondEraseSchedule.push(Partitions.begin() + i); } } @@ -1639,90 +1639,90 @@ TChannelsMapping GetPoolsMapping(const TChannelsBindings& bindings) { return mapping; } -TString TExportInfo::ToString() const { - return TStringBuilder() << "{" - << " Id: " << Id - << " Uid: '" << Uid << "'" - << " Kind: " << Kind - << " DomainPathId: " << DomainPathId - << " ExportPathId: " << ExportPathId - << " UserSID: '" << UserSID << "'" - << " State: " << State - << " WaitTxId: " << WaitTxId - << " Issue: '" << Issue << "'" - << " Items: " << Items.size() - << " PendingItems: " << PendingItems.size() - << " PendingDropItems: " << PendingDropItems.size() - << " }"; -} - -TString TExportInfo::TItem::ToString(ui32 idx) const { - return TStringBuilder() << "{" - << " Idx: " << idx - << " SourcePathName: '" << SourcePathName << "'" - << " SourcePathId: " << SourcePathId - << " State: " << State - << " SubState: " << SubState - << " WaitTxId: " << WaitTxId - << " Issue: '" << Issue << "'" - << " }"; -} - -bool TExportInfo::TItem::IsDone(const TExportInfo::TItem& item) { - return item.State == EState::Done; -} - -bool TExportInfo::TItem::IsDropped(const TExportInfo::TItem& item) { - return item.State == EState::Dropped; -} - -bool TExportInfo::AllItemsAreDropped() const { - return AllOf(Items, &TExportInfo::TItem::IsDropped); -} - +TString TExportInfo::ToString() const { + return TStringBuilder() << "{" + << " Id: " << Id + << " Uid: '" << Uid << "'" + << " Kind: " << Kind + << " DomainPathId: " << DomainPathId + << " ExportPathId: " << ExportPathId + << " UserSID: '" << UserSID << "'" + << " State: " << State + << " WaitTxId: " << WaitTxId + << " Issue: '" << Issue << "'" + << " Items: " << Items.size() + << " PendingItems: " << PendingItems.size() + << " PendingDropItems: " << PendingDropItems.size() + << " }"; +} + +TString TExportInfo::TItem::ToString(ui32 idx) const { + return TStringBuilder() << "{" + << " Idx: " << idx + << " SourcePathName: '" << SourcePathName << "'" + << " SourcePathId: " << SourcePathId + << " State: " << State + << " SubState: " << SubState + << " WaitTxId: " << WaitTxId + << " Issue: '" << Issue << "'" + << " }"; +} + +bool TExportInfo::TItem::IsDone(const TExportInfo::TItem& item) { + return item.State == EState::Done; +} + +bool TExportInfo::TItem::IsDropped(const TExportInfo::TItem& item) { + return item.State == EState::Dropped; +} + +bool TExportInfo::AllItemsAreDropped() const { + return AllOf(Items, &TExportInfo::TItem::IsDropped); +} + void TExportInfo::AddNotifySubscriber(const TActorId &actorId) { Y_VERIFY(!IsFinished()); Subscribers.insert(actorId); } -TString TImportInfo::ToString() const { - return TStringBuilder() << "{" - << " Id: " << Id - << " Uid: '" << Uid << "'" - << " Kind: " << Kind - << " DomainPathId: " << DomainPathId - << " UserSID: '" << UserSID << "'" - << " State: " << State - << " Issue: '" << Issue << "'" - << " Items: " << Items.size() - << " }"; -} - -TString TImportInfo::TItem::ToString(ui32 idx) const { - return TStringBuilder() << "{" - << " Idx: " << idx - << " DstPathName: '" << DstPathName << "'" - << " DstPathId: " << DstPathId - << " State: " << State - << " SubState: " << SubState - << " WaitTxId: " << WaitTxId - << " Issue: '" << Issue << "'" - << " }"; -} - -bool TImportInfo::TItem::IsDone(const TImportInfo::TItem& item) { - return item.State == EState::Done; -} - -bool TImportInfo::IsFinished() const { - return State == EState::Done || State == EState::Cancelled; -} - -void TImportInfo::AddNotifySubscriber(const TActorId &actorId) { - Y_VERIFY(!IsFinished()); - Subscribers.insert(actorId); -} - +TString TImportInfo::ToString() const { + return TStringBuilder() << "{" + << " Id: " << Id + << " Uid: '" << Uid << "'" + << " Kind: " << Kind + << " DomainPathId: " << DomainPathId + << " UserSID: '" << UserSID << "'" + << " State: " << State + << " Issue: '" << Issue << "'" + << " Items: " << Items.size() + << " }"; +} + +TString TImportInfo::TItem::ToString(ui32 idx) const { + return TStringBuilder() << "{" + << " Idx: " << idx + << " DstPathName: '" << DstPathName << "'" + << " DstPathId: " << DstPathId + << " State: " << State + << " SubState: " << SubState + << " WaitTxId: " << WaitTxId + << " Issue: '" << Issue << "'" + << " }"; +} + +bool TImportInfo::TItem::IsDone(const TImportInfo::TItem& item) { + return item.State == EState::Done; +} + +bool TImportInfo::IsFinished() const { + return State == EState::Done || State == EState::Cancelled; +} + +void TImportInfo::AddNotifySubscriber(const TActorId &actorId) { + Y_VERIFY(!IsFinished()); + Subscribers.insert(actorId); +} + TIndexBuildInfo::TShardStatus::TShardStatus() { TCell cell; TVector<TCell> vec = {cell}; @@ -1850,44 +1850,44 @@ const TString &TColumnFamiliesMerger::CanonizeName(const TString &familyName) { return familyName; } -void TPQShardInfo::TKeyRange::SerializeToProto(NKikimrPQ::TPartitionKeyRange& proto) const { - if (FromBound) { - proto.SetFromBound(*FromBound); - } - - if (ToBound) { - proto.SetToBound(*ToBound); - } -} - -bool TPersQueueGroupInfo::FillKeySchema(const NKikimrPQ::TPQTabletConfig& tabletConfig, TString& error) { - KeySchema.clear(); - KeySchema.reserve(tabletConfig.PartitionKeySchemaSize()); - - for (const auto& component : tabletConfig.GetPartitionKeySchema()) { - if (!NScheme::NTypeIds::IsYqlType(component.GetTypeId())) { - error = TStringBuilder() << "TypeId is not supported" - << ": typeId# " << component.GetTypeId() - << ", component# " << component.GetName(); - return false; - } - - KeySchema.push_back(component.GetTypeId()); - } - - return true; -} - -bool TPersQueueGroupInfo::FillKeySchema(const TString& tabletConfig) { - NKikimrPQ::TPQTabletConfig proto; - if (!proto.ParseFromString(tabletConfig)) { - return false; - } - - TString unused; - return FillKeySchema(proto, unused); -} - +void TPQShardInfo::TKeyRange::SerializeToProto(NKikimrPQ::TPartitionKeyRange& proto) const { + if (FromBound) { + proto.SetFromBound(*FromBound); + } + + if (ToBound) { + proto.SetToBound(*ToBound); + } +} + +bool TPersQueueGroupInfo::FillKeySchema(const NKikimrPQ::TPQTabletConfig& tabletConfig, TString& error) { + KeySchema.clear(); + KeySchema.reserve(tabletConfig.PartitionKeySchemaSize()); + + for (const auto& component : tabletConfig.GetPartitionKeySchema()) { + if (!NScheme::NTypeIds::IsYqlType(component.GetTypeId())) { + error = TStringBuilder() << "TypeId is not supported" + << ": typeId# " << component.GetTypeId() + << ", component# " << component.GetName(); + return false; + } + + KeySchema.push_back(component.GetTypeId()); + } + + return true; +} + +bool TPersQueueGroupInfo::FillKeySchema(const TString& tabletConfig) { + NKikimrPQ::TPQTabletConfig proto; + if (!proto.ParseFromString(tabletConfig)) { + return false; + } + + TString unused; + return FillKeySchema(proto, unused); +} + TBillingStats::TBillingStats(ui64 rows, ui64 bytes) : Rows(rows) , Bytes(bytes) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 568fb2a568a..2e5710c3d3e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -26,8 +26,8 @@ #include <google/protobuf/util/message_differencer.h> #include <util/generic/ptr.h> -#include <util/generic/queue.h> -#include <util/generic/vector.h> +#include <util/generic/queue.h> +#include <util/generic/vector.h> namespace NKikimr { namespace NSchemeShard { @@ -85,18 +85,18 @@ TChannelsMapping GetPoolsMapping(const TChannelsBindings& bindings); struct TTableShardInfo { TShardIdx ShardIdx = InvalidShardIdx; TString EndOfRange; - TInstant LastCondErase; - TInstant NextCondErase; - mutable TMaybe<TDuration> LastCondEraseLag; + TInstant LastCondErase; + TInstant NextCondErase; + mutable TMaybe<TDuration> LastCondEraseLag; // TODO: remove this ctor. It's used for vector.resize() that is not clear. TTableShardInfo() = default; - TTableShardInfo(const TShardIdx& idx, TString rangeEnd, ui64 lastCondErase = 0, ui64 nextCondErase = 0) + TTableShardInfo(const TShardIdx& idx, TString rangeEnd, ui64 lastCondErase = 0, ui64 nextCondErase = 0) : ShardIdx(idx) , EndOfRange(rangeEnd) - , LastCondErase(TInstant::FromValue(lastCondErase)) - , NextCondErase(TInstant::FromValue(nextCondErase)) + , LastCondErase(TInstant::FromValue(lastCondErase)) + , NextCondErase(TInstant::FromValue(nextCondErase)) {} }; @@ -165,8 +165,8 @@ struct TPartitionConfigMerger { }; -struct TSubDomainInfo; - +struct TSubDomainInfo; + struct TTableInfo : public TSimpleRefCount<TTableInfo> { using TPtr = TIntrusivePtr<TTableInfo>; using TCPtr = TIntrusiveConstPtr<TTableInfo>; @@ -194,17 +194,17 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { bool IsDropped() const { return DeleteVersion != Max<ui64>(); } }; - struct TBackupRestoreResult { - enum class EKind: ui8 { - Backup = 0, - Restore, - }; - - ui64 StartDateTime; // seconds - ui64 CompletionDateTime; // seconds + struct TBackupRestoreResult { + enum class EKind: ui8 { + Backup = 0, + Restore, + }; + + ui64 StartDateTime; // seconds + ui64 CompletionDateTime; // seconds ui32 TotalShardCount; - ui32 SuccessShardCount; - THashMap<TShardIdx, TTxState::TShardStatus> ShardStatuses; + ui32 SuccessShardCount; + THashMap<TShardIdx, TTxState::TShardStatus> ShardStatuses; ui64 DataTotalSize; }; @@ -318,7 +318,7 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { ui64 AlterVersion = 0; THashMap<ui32, TColumn> Columns; TVector<ui32> KeyColumnIds; - bool IsBackup = false; + bool IsBackup = false; NKikimrSchemeOp::TTableDescription TableDescriptionDiff; TMaybeFail<NKikimrSchemeOp::TTableDescription> TableDescriptionFull; @@ -358,10 +358,10 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { ui32 NextColumnId = 1; // Next unallocated column id ui64 AlterVersion = 0; - ui64 PartitioningVersion = 0; + ui64 PartitioningVersion = 0; THashMap<ui32, TColumn> Columns; TVector<ui32> KeyColumnIds; - bool IsBackup = false; + bool IsBackup = false; TAlterTableInfo::TPtr AlterData; @@ -369,9 +369,9 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { NKikimrSchemeOp::TBackupTask BackupSettings; NKikimrSchemeOp::TRestoreTask RestoreSettings; - TMap<TTxId, TBackupRestoreResult> BackupHistory; - TMap<TTxId, TBackupRestoreResult> RestoreHistory; - + TMap<TTxId, TBackupRestoreResult> BackupHistory; + TMap<TTxId, TBackupRestoreResult> RestoreHistory; + TString PreSerializedPathDescription; THashMap<TShardIdx, NKikimrSchemeOp::TPartitionConfig> PerShardPartitionConfig; @@ -379,52 +379,52 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { const NKikimrSchemeOp::TPartitionConfig& PartitionConfig() const { return TableDescription.GetPartitionConfig(); } NKikimrSchemeOp::TPartitionConfig& MutablePartitionConfig() { return *TableDescription.MutablePartitionConfig(); } - bool HasTTLSettings() const { return TableDescription.HasTTLSettings(); } + bool HasTTLSettings() const { return TableDescription.HasTTLSettings(); } const NKikimrSchemeOp::TTTLSettings& TTLSettings() const { return TableDescription.GetTTLSettings(); } - bool IsTTLEnabled() const { return HasTTLSettings() && TTLSettings().HasEnabled(); } - + bool IsTTLEnabled() const { return HasTTLSettings() && TTLSettings().HasEnabled(); } + NKikimrSchemeOp::TTTLSettings& MutableTTLSettings() { - TTLColumnId.Clear(); - return *TableDescription.MutableTTLSettings(); - } - - ui32 GetTTLColumnId() const { - if (!IsTTLEnabled()) { - return Max<ui32>(); - } - - if (!TTLColumnId) { - for (const auto& [id, col] : Columns) { - if (!col.IsDropped() && col.Name == TTLSettings().GetEnabled().GetColumnName()) { - TTLColumnId = id; - break; - } - } - } - - if (!TTLColumnId) { - TTLColumnId = Max<ui32>(); - } - - return *TTLColumnId; - } - + TTLColumnId.Clear(); + return *TableDescription.MutableTTLSettings(); + } + + ui32 GetTTLColumnId() const { + if (!IsTTLEnabled()) { + return Max<ui32>(); + } + + if (!TTLColumnId) { + for (const auto& [id, col] : Columns) { + if (!col.IsDropped() && col.Name == TTLSettings().GetEnabled().GetColumnName()) { + TTLColumnId = id; + break; + } + } + } + + if (!TTLColumnId) { + TTLColumnId = Max<ui32>(); + } + + return *TTLColumnId; + } + private: - using TPartitionsVec = TVector<TTableShardInfo>; - - struct TSortByNextCondErase { - using TIterator = TPartitionsVec::iterator; - - bool operator()(TIterator left, TIterator right) const { - return left->NextCondErase > right->NextCondErase; - } - }; - - TPartitionsVec Partitions; + using TPartitionsVec = TVector<TTableShardInfo>; + + struct TSortByNextCondErase { + using TIterator = TPartitionsVec::iterator; + + bool operator()(TIterator left, TIterator right) const { + return left->NextCondErase > right->NextCondErase; + } + }; + + TPartitionsVec Partitions; THashMap<TShardIdx, ui64> Shard2PartitionIdx; // shardIdx -> index in Partitions - TPriorityQueue<TPartitionsVec::iterator, TVector<TPartitionsVec::iterator>, TSortByNextCondErase> CondEraseSchedule; - THashSet<TShardIdx> InFlightCondErase; - mutable TMaybe<ui32> TTLColumnId; + TPriorityQueue<TPartitionsVec::iterator, TVector<TPartitionsVec::iterator>, TSortByNextCondErase> CondEraseSchedule; + THashSet<TShardIdx> InFlightCondErase; + mutable TMaybe<ui32> TTLColumnId; THashSet<TOperationId> SplitOpsInFlight; THashMap<TOperationId, TVector<TShardIdx>> ShardsInSplitMergeByOpId; THashMap<TShardIdx, TOperationId> ShardsInSplitMergeByShards; @@ -432,20 +432,20 @@ private: TStats Stats; bool ShardsStatsDetached = false; - TPartitionsVec::iterator FindPartition(const TShardIdx& shardIdx) { - auto it = Shard2PartitionIdx.find(shardIdx); - if (it == Shard2PartitionIdx.end()) { - return Partitions.end(); - } - - const auto partitionIdx = it->second; - if (partitionIdx >= Partitions.size()) { - return Partitions.end(); - } - - return Partitions.begin() + partitionIdx; - } - + TPartitionsVec::iterator FindPartition(const TShardIdx& shardIdx) { + auto it = Shard2PartitionIdx.find(shardIdx); + if (it == Shard2PartitionIdx.end()) { + return Partitions.end(); + } + + const auto partitionIdx = it->second; + if (partitionIdx >= Partitions.size()) { + return Partitions.end(); + } + + return Partitions.begin() + partitionIdx; + } + public: TTableInfo() = default; @@ -454,7 +454,7 @@ public: , AlterVersion(alterData.AlterVersion) , Columns(std::move(alterData.Columns)) , KeyColumnIds(std::move(alterData.KeyColumnIds)) - , IsBackup(alterData.IsBackup) + , IsBackup(alterData.IsBackup) { TableDescription.Swap(alterData.TableDescriptionFull.Get()); } @@ -463,8 +463,8 @@ public: TPtr source, NKikimrSchemeOp::TTableDescription& descr, const NScheme::TTypeRegistry& typeRegistry, - const TSchemeLimits& limits, const TSubDomainInfo& subDomain, - TString& errStr, const THashSet<TString>& localSequences = {}); + const TSchemeLimits& limits, const TSubDomainInfo& subDomain, + TString& errStr, const THashSet<TString>& localSequences = {}); static ui32 ShardsToCreate(const NKikimrSchemeOp::TTableDescription& descr) { if (descr.HasUniformPartitionsCount()) { @@ -639,50 +639,50 @@ public: PartitionConfig().GetFollowerGroups(), AlterData->PartitionConfigCompatible().GetFollowerGroups()); } - - const TTableShardInfo* GetScheduledCondEraseShard() const { - if (CondEraseSchedule.empty()) { - return nullptr; - } - - return CondEraseSchedule.top(); - } - - const THashSet<TShardIdx>& GetInFlightCondErase() const { - return InFlightCondErase; - } - - void AddInFlightCondErase(const TShardIdx& shardIdx) { - const auto* shardInfo = GetScheduledCondEraseShard(); - Y_VERIFY(shardInfo && shardIdx == shardInfo->ShardIdx); - - InFlightCondErase.insert(shardIdx); - CondEraseSchedule.pop(); - } - - void RescheduleCondErase(const TShardIdx& shardIdx) { - Y_VERIFY(InFlightCondErase.contains(shardIdx)); - - auto it = FindPartition(shardIdx); - Y_VERIFY(it != Partitions.end()); - - CondEraseSchedule.push(it); - InFlightCondErase.erase(shardIdx); - } - - void ScheduleNextCondErase(const TShardIdx& shardIdx, const TInstant& now, const TDuration& next) { - Y_VERIFY(InFlightCondErase.contains(shardIdx)); - - auto it = FindPartition(shardIdx); - Y_VERIFY(it != Partitions.end()); - - it->LastCondErase = now; - it->NextCondErase = now + next; - it->LastCondEraseLag = TDuration::Zero(); - - CondEraseSchedule.push(it); - InFlightCondErase.erase(shardIdx); - } + + const TTableShardInfo* GetScheduledCondEraseShard() const { + if (CondEraseSchedule.empty()) { + return nullptr; + } + + return CondEraseSchedule.top(); + } + + const THashSet<TShardIdx>& GetInFlightCondErase() const { + return InFlightCondErase; + } + + void AddInFlightCondErase(const TShardIdx& shardIdx) { + const auto* shardInfo = GetScheduledCondEraseShard(); + Y_VERIFY(shardInfo && shardIdx == shardInfo->ShardIdx); + + InFlightCondErase.insert(shardIdx); + CondEraseSchedule.pop(); + } + + void RescheduleCondErase(const TShardIdx& shardIdx) { + Y_VERIFY(InFlightCondErase.contains(shardIdx)); + + auto it = FindPartition(shardIdx); + Y_VERIFY(it != Partitions.end()); + + CondEraseSchedule.push(it); + InFlightCondErase.erase(shardIdx); + } + + void ScheduleNextCondErase(const TShardIdx& shardIdx, const TInstant& now, const TDuration& next) { + Y_VERIFY(InFlightCondErase.contains(shardIdx)); + + auto it = FindPartition(shardIdx); + Y_VERIFY(it != Partitions.end()); + + it->LastCondErase = now; + it->NextCondErase = now + next; + it->LastCondEraseLag = TDuration::Zero(); + + CondEraseSchedule.push(it); + InFlightCondErase.erase(shardIdx); + } bool IsUsingSequence(const TString& name) { for (const auto& pr : Columns) { @@ -812,20 +812,20 @@ struct TOlapTableInfo : TSimpleRefCount<TOlapTableInfo> { struct TPQShardInfo : TSimpleRefCount<TPQShardInfo> { using TPtr = TIntrusivePtr<TPQShardInfo>; - using TKeySchema = TVector<NScheme::TTypeId>; - - struct TKeyRange { - TMaybe<TString> FromBound; - TMaybe<TString> ToBound; - - void SerializeToProto(NKikimrPQ::TPartitionKeyRange& proto) const; - }; - + using TKeySchema = TVector<NScheme::TTypeId>; + + struct TKeyRange { + TMaybe<TString> FromBound; + TMaybe<TString> ToBound; + + void SerializeToProto(NKikimrPQ::TPartitionKeyRange& proto) const; + }; + struct TPersQueueInfo { ui32 PqId = 0; ui32 GroupId = 0; ui64 AlterVersion = 0; - TMaybe<TKeyRange> KeyRange; + TMaybe<TKeyRange> KeyRange; }; TVector<TPersQueueInfo> PQInfos; @@ -940,54 +940,54 @@ struct TShardInfo { static TShardInfo SequenceShardInfo(TTxId txId, TPathId pathId) { return TShardInfo(txId, pathId, ETabletType::SequenceShard); } - - static TShardInfo ReplicationControllerInfo(TTxId txId, TPathId pathId) { - return TShardInfo(txId, pathId, ETabletType::ReplicationController); - } + + static TShardInfo ReplicationControllerInfo(TTxId txId, TPathId pathId) { + return TShardInfo(txId, pathId, ETabletType::ReplicationController); + } }; struct TPersQueueGroupInfo : TSimpleRefCount<TPersQueueGroupInfo> { using TPtr = TIntrusivePtr<TPersQueueGroupInfo>; - using TKeySchema = TPQShardInfo::TKeySchema; - - struct TPartitionToAdd { - using TKeyRange = TPQShardInfo::TKeyRange; - - ui32 PartitionId; - ui32 GroupId; - TMaybe<TKeyRange> KeyRange; - - explicit TPartitionToAdd(ui32 partitionId, ui32 groupId, const TMaybe<TKeyRange>& keyRange = Nothing()) - : PartitionId(partitionId) - , GroupId(groupId) - , KeyRange(keyRange) - { - } - - bool operator==(const TPartitionToAdd& rhs) const { - return PartitionId == rhs.PartitionId - && GroupId == rhs.GroupId; - } - - struct THash { - inline size_t operator()(const TPartitionToAdd& obj) const { - const ::THash<ui32> hashFn; - return CombineHashes(hashFn(obj.PartitionId), hashFn(obj.GroupId)); - } - }; - }; - + using TKeySchema = TPQShardInfo::TKeySchema; + + struct TPartitionToAdd { + using TKeyRange = TPQShardInfo::TKeyRange; + + ui32 PartitionId; + ui32 GroupId; + TMaybe<TKeyRange> KeyRange; + + explicit TPartitionToAdd(ui32 partitionId, ui32 groupId, const TMaybe<TKeyRange>& keyRange = Nothing()) + : PartitionId(partitionId) + , GroupId(groupId) + , KeyRange(keyRange) + { + } + + bool operator==(const TPartitionToAdd& rhs) const { + return PartitionId == rhs.PartitionId + && GroupId == rhs.GroupId; + } + + struct THash { + inline size_t operator()(const TPartitionToAdd& obj) const { + const ::THash<ui32> hashFn; + return CombineHashes(hashFn(obj.PartitionId), hashFn(obj.GroupId)); + } + }; + }; + ui64 TotalGroupCount = 0; ui64 TotalPartitionCount = 0; ui32 NextPartitionId = 0; - THashSet<TPartitionToAdd, TPartitionToAdd::THash> PartitionsToAdd; + THashSet<TPartitionToAdd, TPartitionToAdd::THash> PartitionsToAdd; THashSet<ui32> PartitionsToDelete; ui32 MaxPartsPerTablet = 0; ui64 AlterVersion = 0; TString TabletConfig; - TString BootstrapConfig; + TString BootstrapConfig; THashMap<TShardIdx, TPQShardInfo::TPtr> Shards; // key - shardIdx - TKeySchema KeySchema; + TKeySchema KeySchema; TPersQueueGroupInfo::TPtr AlterData; // changes to be applyed TTabletId BalancerTabletID = InvalidTabletId; TShardIdx BalancerShardIdx = InvalidShardIdx; @@ -995,9 +995,9 @@ struct TPersQueueGroupInfo : TSimpleRefCount<TPersQueueGroupInfo> { TString PreSerializedPathDescription; // Cached path description TString PreSerializedPartitionsDescription; // Cached partition description - bool FillKeySchema(const NKikimrPQ::TPQTabletConfig& tabletConfig, TString& error); - bool FillKeySchema(const TString& tabletConfig); - + bool FillKeySchema(const NKikimrPQ::TPQTabletConfig& tabletConfig, TString& error); + bool FillKeySchema(const TString& tabletConfig); + bool HasBalancer() const { return bool(BalancerTabletID); } ui32 GetTotalPartitionCountWithAlter() const { @@ -1032,7 +1032,7 @@ struct TPersQueueGroupInfo : TSimpleRefCount<TPersQueueGroupInfo> { Y_VERIFY(alterData->TotalPartitionCount); Y_VERIFY(alterData->NextPartitionId); Y_VERIFY(alterData->MaxPartsPerTablet); - alterData->KeySchema = KeySchema; + alterData->KeySchema = KeySchema; alterData->BalancerTabletID = BalancerTabletID; alterData->BalancerShardIdx = BalancerShardIdx; AlterData = alterData; @@ -1050,7 +1050,7 @@ struct TPersQueueGroupInfo : TSimpleRefCount<TPersQueueGroupInfo> { Y_VERIFY(BalancerTabletID == AlterData->BalancerTabletID || !HasBalancer()); Y_VERIFY(AlterData->HasBalancer()); Y_VERIFY(AlterData->BalancerShardIdx); - KeySchema = AlterData->KeySchema; + KeySchema = AlterData->KeySchema; BalancerTabletID = AlterData->BalancerTabletID; BalancerShardIdx = AlterData->BalancerShardIdx; AlterData.Reset(); @@ -1134,14 +1134,14 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { using TPtr = TIntrusivePtr<TSubDomainInfo>; using TConstPtr = TIntrusiveConstPtr<TSubDomainInfo>; - struct TDiskSpaceUsage { - struct TTables { - ui64 TotalSize = 0; - ui64 DataSize = 0; - ui64 IndexSize = 0; - } Tables; - }; - + struct TDiskSpaceUsage { + struct TTables { + ui64 TotalSize = 0; + ui64 DataSize = 0; + ui64 IndexSize = 0; + } Tables; + }; + struct TDiskSpaceQuotas { ui64 HardQuota; ui64 SoftQuota; @@ -1152,18 +1152,18 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { }; TSubDomainInfo() = default; - explicit TSubDomainInfo(ui64 version, const TPathId& resourcesDomainId) + explicit TSubDomainInfo(ui64 version, const TPathId& resourcesDomainId) { ProcessingParams.SetVersion(version); - ResourcesDomainId = resourcesDomainId; + ResourcesDomainId = resourcesDomainId; } - TSubDomainInfo(ui64 version, ui64 resolution, ui32 bucketsPerMediator, const TPathId& resourcesDomainId) + TSubDomainInfo(ui64 version, ui64 resolution, ui32 bucketsPerMediator, const TPathId& resourcesDomainId) { ProcessingParams.SetVersion(version); ProcessingParams.SetPlanResolution(resolution); ProcessingParams.SetTimeCastBucketsPerMediator(bucketsPerMediator); - ResourcesDomainId = resourcesDomainId; + ResourcesDomainId = resourcesDomainId; } TSubDomainInfo(const TSubDomainInfo& other @@ -1226,22 +1226,22 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { ProcessingParams.SetVersion(subDomainVersion); } - TPathId GetResourcesDomainId() const { - return ResourcesDomainId; - } - - void SetResourcesDomainId(const TPathId& domainId) { - ResourcesDomainId = domainId; - } - - TTabletId GetSharedHive() const { - return SharedHive; - } - - void SetSharedHive(const TTabletId& hiveId) { - SharedHive = hiveId; - } - + TPathId GetResourcesDomainId() const { + return ResourcesDomainId; + } + + void SetResourcesDomainId(const TPathId& domainId) { + ResourcesDomainId = domainId; + } + + TTabletId GetSharedHive() const { + return SharedHive; + } + + void SetSharedHive(const TTabletId& hiveId) { + SharedHive = hiveId; + } + ui64 GetPlanResolution() const { return ProcessingParams.GetPlanResolution(); } @@ -1338,7 +1338,7 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { void ActualizeAlterData(const THashMap<TShardIdx, TShardInfo>& allShards, TInstant now, bool isExternal, IQuotaCounters* counters) { Y_VERIFY(AlterData); - + AlterData->SetPathsInside(GetPathsInside()); AlterData->InternalShards.swap(InternalShards); AlterData->Initialize(allShards); @@ -1377,20 +1377,20 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { return DatabaseQuotas ? DatabaseQuotas->data_stream_reserved_storage_quota() : 0; } - TDuration GetTtlMinRunInterval() const { - static constexpr auto TtlMinRunInterval = TDuration::Minutes(15); - - if (!DatabaseQuotas) { - return TtlMinRunInterval; - } - - if (!DatabaseQuotas->ttl_min_run_internal_seconds()) { - return TtlMinRunInterval; - } - - return TDuration::Seconds(DatabaseQuotas->ttl_min_run_internal_seconds()); - } - + TDuration GetTtlMinRunInterval() const { + static constexpr auto TtlMinRunInterval = TDuration::Minutes(15); + + if (!DatabaseQuotas) { + return TtlMinRunInterval; + } + + if (!DatabaseQuotas->ttl_min_run_internal_seconds()) { + return TtlMinRunInterval; + } + + return TDuration::Seconds(DatabaseQuotas->ttl_min_run_internal_seconds()); + } + TDiskSpaceQuotas GetDiskSpaceQuotas() const { ui64 hardQuota = DatabaseQuotas ? DatabaseQuotas->data_size_hard_quota() : 0; ui64 softQuota = DatabaseQuotas ? DatabaseQuotas->data_size_soft_quota() : 0; @@ -1547,20 +1547,20 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { SequenceShards.erase(it); } - const THashSet<TShardIdx>& GetReplicationControllers() const { - return ReplicationControllers; - } - - void AddReplicationController(const TShardIdx& shardIdx) { - ReplicationControllers.insert(shardIdx); - } - - void RemoveReplicationController(const TShardIdx& shardIdx) { - auto it = ReplicationControllers.find(shardIdx); - Y_VERIFY_S(it != ReplicationControllers.end(), "shardIdx: " << shardIdx); - ReplicationControllers.erase(it); - } - + const THashSet<TShardIdx>& GetReplicationControllers() const { + return ReplicationControllers; + } + + void AddReplicationController(const TShardIdx& shardIdx) { + ReplicationControllers.insert(shardIdx); + } + + void RemoveReplicationController(const TShardIdx& shardIdx) { + auto it = ReplicationControllers.find(shardIdx); + Y_VERIFY_S(it != ReplicationControllers.end(), "shardIdx: " << shardIdx); + ReplicationControllers.erase(it); + } + const NKikimrSubDomains::TProcessingParams& GetProcessingParams() const { return ProcessingParams; } @@ -1629,22 +1629,22 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> { } void AggrDiskSpaceUsage(IQuotaCounters* counters, const TTableInfo::TPartitionStats& newAggr, const TTableInfo::TPartitionStats& oldAggr = TTableInfo::TPartitionStats()) { - DiskSpaceUsage.Tables.DataSize += (newAggr.DataSize - oldAggr.DataSize); + DiskSpaceUsage.Tables.DataSize += (newAggr.DataSize - oldAggr.DataSize); counters->ChangeDiskSpaceTablesDataBytes(newAggr.DataSize - oldAggr.DataSize); - DiskSpaceUsage.Tables.IndexSize += (newAggr.IndexSize - oldAggr.IndexSize); + DiskSpaceUsage.Tables.IndexSize += (newAggr.IndexSize - oldAggr.IndexSize); counters->ChangeDiskSpaceTablesIndexBytes(newAggr.IndexSize - oldAggr.IndexSize); i64 oldTotalBytes = DiskSpaceUsage.Tables.TotalSize; - DiskSpaceUsage.Tables.TotalSize = DiskSpaceUsage.Tables.DataSize + DiskSpaceUsage.Tables.IndexSize; + DiskSpaceUsage.Tables.TotalSize = DiskSpaceUsage.Tables.DataSize + DiskSpaceUsage.Tables.IndexSize; i64 newTotalBytes = DiskSpaceUsage.Tables.TotalSize; counters->ChangeDiskSpaceTablesTotalBytes(newTotalBytes - oldTotalBytes); - } - - const TDiskSpaceUsage& GetDiskSpaceUsage() const { - return DiskSpaceUsage; - } - + } + + const TDiskSpaceUsage& GetDiskSpaceUsage() const { + return DiskSpaceUsage; + } + const TMaybe<NKikimrSubDomains::TSchemeQuotas>& GetDeclaredSchemeQuotas() const { return DeclaredSchemeQuotas; } @@ -1793,18 +1793,18 @@ private: TSchemeQuotas SchemeQuotas; ui64 PathsInsideCount = 0; - TDiskSpaceUsage DiskSpaceUsage; + TDiskSpaceUsage DiskSpaceUsage; THashSet<TShardIdx> InternalShards; THashSet<TShardIdx> SequenceShards; - THashSet<TShardIdx> ReplicationControllers; + THashSet<TShardIdx> ReplicationControllers; ui64 PQPartitionsInsideCount = 0; ui64 PQReservedStorage = 0; - TPathId ResourcesDomainId; - TTabletId SharedHive = InvalidTabletId; - + TPathId ResourcesDomainId; + TTabletId SharedHive = InvalidTabletId; + NLoginProto::TSecurityState SecurityState; ui64 SecurityStateVersion = 0; @@ -2099,46 +2099,46 @@ struct TTableIndexInfo : public TSimpleRefCount<TTableIndexInfo> { TTableIndexInfo::TPtr AlterData = nullptr; }; -struct TCdcStreamInfo : public TSimpleRefCount<TCdcStreamInfo> { - using TPtr = TIntrusivePtr<TCdcStreamInfo>; +struct TCdcStreamInfo : public TSimpleRefCount<TCdcStreamInfo> { + using TPtr = TIntrusivePtr<TCdcStreamInfo>; using EMode = NKikimrSchemeOp::ECdcStreamMode; using EState = NKikimrSchemeOp::ECdcStreamState; - - TCdcStreamInfo(ui64 version, EMode mode, EState state) - : AlterVersion(version) - , Mode(mode) - , State(state) - {} - - TCdcStreamInfo(const TCdcStreamInfo&) = default; - - TPtr CreateNextVersion() { - Y_VERIFY(AlterData == nullptr); - TPtr result = new TCdcStreamInfo(*this); - ++result->AlterVersion; - this->AlterData = result; - return result; - } - - static TPtr New(EMode mode) { - return new TCdcStreamInfo(0, mode, EState::ECdcStreamStateInvalid); - } - + + TCdcStreamInfo(ui64 version, EMode mode, EState state) + : AlterVersion(version) + , Mode(mode) + , State(state) + {} + + TCdcStreamInfo(const TCdcStreamInfo&) = default; + + TPtr CreateNextVersion() { + Y_VERIFY(AlterData == nullptr); + TPtr result = new TCdcStreamInfo(*this); + ++result->AlterVersion; + this->AlterData = result; + return result; + } + + static TPtr New(EMode mode) { + return new TCdcStreamInfo(0, mode, EState::ECdcStreamStateInvalid); + } + static TPtr Create(const NKikimrSchemeOp::TCdcStreamDescription& desc) { - TPtr result = New(desc.GetMode()); - TPtr alterData = result->CreateNextVersion(); - alterData->State = EState::ECdcStreamStateReady; - - return result; - } - - ui64 AlterVersion = 1; - EMode Mode; - EState State; - - TCdcStreamInfo::TPtr AlterData = nullptr; -}; - + TPtr result = New(desc.GetMode()); + TPtr alterData = result->CreateNextVersion(); + alterData->State = EState::ECdcStreamStateReady; + + return result; + } + + ui64 AlterVersion = 1; + EMode Mode; + EState State; + + TCdcStreamInfo::TPtr AlterData = nullptr; +}; + struct TSequenceInfo : public TSimpleRefCount<TSequenceInfo> { using TPtr = TIntrusivePtr<TSequenceInfo>; @@ -2169,289 +2169,289 @@ struct TSequenceInfo : public TSimpleRefCount<TSequenceInfo> { ui64 SequenceShard = 0; }; -struct TReplicationInfo : public TSimpleRefCount<TReplicationInfo> { - using TPtr = TIntrusivePtr<TReplicationInfo>; - - TReplicationInfo(ui64 alterVersion) - : AlterVersion(alterVersion) - { - } - - TReplicationInfo(ui64 alterVersion, NKikimrSchemeOp::TReplicationDescription&& desc) - : AlterVersion(alterVersion) - , Description(std::move(desc)) - { - } - - TPtr CreateNextVersion() { - Y_VERIFY(AlterData == nullptr); - - TPtr result = new TReplicationInfo(*this); - ++result->AlterVersion; - this->AlterData = result; - - return result; - } - - static TPtr New() { - return new TReplicationInfo(0); - } - - static TPtr Create(NKikimrSchemeOp::TReplicationDescription&& desc) { - TPtr result = New(); - TPtr alterData = result->CreateNextVersion(); - alterData->Description = std::move(desc); - - return result; - } - - ui64 AlterVersion = 0; - TReplicationInfo::TPtr AlterData = nullptr; - NKikimrSchemeOp::TReplicationDescription Description; -}; - -struct TPublicationInfo { - TSet<std::pair<TPathId, ui64>> Paths; +struct TReplicationInfo : public TSimpleRefCount<TReplicationInfo> { + using TPtr = TIntrusivePtr<TReplicationInfo>; + + TReplicationInfo(ui64 alterVersion) + : AlterVersion(alterVersion) + { + } + + TReplicationInfo(ui64 alterVersion, NKikimrSchemeOp::TReplicationDescription&& desc) + : AlterVersion(alterVersion) + , Description(std::move(desc)) + { + } + + TPtr CreateNextVersion() { + Y_VERIFY(AlterData == nullptr); + + TPtr result = new TReplicationInfo(*this); + ++result->AlterVersion; + this->AlterData = result; + + return result; + } + + static TPtr New() { + return new TReplicationInfo(0); + } + + static TPtr Create(NKikimrSchemeOp::TReplicationDescription&& desc) { + TPtr result = New(); + TPtr alterData = result->CreateNextVersion(); + alterData->Description = std::move(desc); + + return result; + } + + ui64 AlterVersion = 0; + TReplicationInfo::TPtr AlterData = nullptr; + NKikimrSchemeOp::TReplicationDescription Description; +}; + +struct TPublicationInfo { + TSet<std::pair<TPathId, ui64>> Paths; THashSet<TActorId> Subscribers; -}; - -// namespace NExport { -struct TExportInfo: public TSimpleRefCount<TExportInfo> { - using TPtr = TIntrusivePtr<TExportInfo>; - - enum class EState: ui8 { - Invalid = 0, - Waiting, - CreateExportDir, - CopyTables, - Transferring, - Done = 240, - Dropping = 241, - Dropped = 242, - Cancellation = 250, - Cancelled = 251, - }; - - enum class EKind: ui8 { - YT = 0, - S3, - }; - - struct TItem { - enum class ESubState: ui8 { - AllocateTxId = 0, - Proposed, - Subscribed, - }; - - TString SourcePathName; - TPathId SourcePathId; - - EState State = EState::Waiting; - ESubState SubState = ESubState::AllocateTxId; - TTxId WaitTxId = InvalidTxId; - TString Issue; - - TItem() = default; - - explicit TItem(const TString& sourcePathName, const TPathId sourcePathId) - : SourcePathName(sourcePathName) - , SourcePathId(sourcePathId) - { - } - - TString ToString(ui32 idx) const; - - static bool IsDone(const TItem& item); - static bool IsDropped(const TItem& item); - }; - - ui64 Id; - TString Uid; - EKind Kind; - TString Settings; - TPathId DomainPathId; - TMaybe<TString> UserSID; - TVector<TItem> Items; - - TPathId ExportPathId = InvalidPathId; - EState State = EState::Invalid; - TTxId WaitTxId = InvalidTxId; - TString Issue; - - TDeque<ui32> PendingItems; - TDeque<ui32> PendingDropItems; - +}; + +// namespace NExport { +struct TExportInfo: public TSimpleRefCount<TExportInfo> { + using TPtr = TIntrusivePtr<TExportInfo>; + + enum class EState: ui8 { + Invalid = 0, + Waiting, + CreateExportDir, + CopyTables, + Transferring, + Done = 240, + Dropping = 241, + Dropped = 242, + Cancellation = 250, + Cancelled = 251, + }; + + enum class EKind: ui8 { + YT = 0, + S3, + }; + + struct TItem { + enum class ESubState: ui8 { + AllocateTxId = 0, + Proposed, + Subscribed, + }; + + TString SourcePathName; + TPathId SourcePathId; + + EState State = EState::Waiting; + ESubState SubState = ESubState::AllocateTxId; + TTxId WaitTxId = InvalidTxId; + TString Issue; + + TItem() = default; + + explicit TItem(const TString& sourcePathName, const TPathId sourcePathId) + : SourcePathName(sourcePathName) + , SourcePathId(sourcePathId) + { + } + + TString ToString(ui32 idx) const; + + static bool IsDone(const TItem& item); + static bool IsDropped(const TItem& item); + }; + + ui64 Id; + TString Uid; + EKind Kind; + TString Settings; + TPathId DomainPathId; + TMaybe<TString> UserSID; + TVector<TItem> Items; + + TPathId ExportPathId = InvalidPathId; + EState State = EState::Invalid; + TTxId WaitTxId = InvalidTxId; + TString Issue; + + TDeque<ui32> PendingItems; + TDeque<ui32> PendingDropItems; + TSet<TActorId> Subscribers; - - explicit TExportInfo( - const ui64 id, - const TString& uid, - const EKind kind, - const TString& settings, - const TPathId domainPathId) - : Id(id) - , Uid(uid) - , Kind(kind) - , Settings(settings) - , DomainPathId(domainPathId) - { - } - - template <typename TSettingsPB> - explicit TExportInfo( - const ui64 id, - const TString& uid, - const EKind kind, - const TSettingsPB& settingsPb, - const TPathId domainPathId) - : TExportInfo(id, uid, kind, SerializeSettings(settingsPb), domainPathId) - { - } - - bool IsValid() const { - return State != EState::Invalid; - } - - bool IsPreparing() const { - return State == EState::CreateExportDir || State == EState::CopyTables; - } - - bool IsWorking() const { - return State == EState::Transferring; - } - - bool IsDropping() const { - return State == EState::Dropping; - } - - bool IsCancelling() const { - return State == EState::Cancellation; - } - - bool IsInProgress() const { - return IsPreparing() || IsWorking() || IsDropping() || IsCancelling(); - } - - bool IsDone() const { - return State == EState::Done; - } - - bool IsCancelled() const { - return State == EState::Cancelled; - } - - bool IsFinished() const { - return IsDone() || IsCancelled(); - } - - bool AllItemsAreDropped() const; + + explicit TExportInfo( + const ui64 id, + const TString& uid, + const EKind kind, + const TString& settings, + const TPathId domainPathId) + : Id(id) + , Uid(uid) + , Kind(kind) + , Settings(settings) + , DomainPathId(domainPathId) + { + } + + template <typename TSettingsPB> + explicit TExportInfo( + const ui64 id, + const TString& uid, + const EKind kind, + const TSettingsPB& settingsPb, + const TPathId domainPathId) + : TExportInfo(id, uid, kind, SerializeSettings(settingsPb), domainPathId) + { + } + + bool IsValid() const { + return State != EState::Invalid; + } + + bool IsPreparing() const { + return State == EState::CreateExportDir || State == EState::CopyTables; + } + + bool IsWorking() const { + return State == EState::Transferring; + } + + bool IsDropping() const { + return State == EState::Dropping; + } + + bool IsCancelling() const { + return State == EState::Cancellation; + } + + bool IsInProgress() const { + return IsPreparing() || IsWorking() || IsDropping() || IsCancelling(); + } + + bool IsDone() const { + return State == EState::Done; + } + + bool IsCancelled() const { + return State == EState::Cancelled; + } + + bool IsFinished() const { + return IsDone() || IsCancelled(); + } + + bool AllItemsAreDropped() const; void AddNotifySubscriber(const TActorId& actorId); - TString ToString() const; - -private: - template <typename TSettingsPB> - static TString SerializeSettings(const TSettingsPB& settings) { - TString serialized; + TString ToString() const; + +private: + template <typename TSettingsPB> + static TString SerializeSettings(const TSettingsPB& settings) { + TString serialized; Y_PROTOBUF_SUPPRESS_NODISCARD settings.SerializeToString(&serialized); - return serialized; - } - -}; // TExportInfo -// } // NExport - -// namespace NImport { -struct TImportInfo: public TSimpleRefCount<TImportInfo> { - using TPtr = TIntrusivePtr<TImportInfo>; - - enum class EState: ui8 { - Invalid = 0, - Waiting, - GetScheme, - CreateTable, - Transferring, - BuildIndexes, - Done = 240, - Cancellation = 250, - Cancelled = 251, - }; - - enum class EKind: ui8 { - S3 = 0, - }; - - struct TItem { - enum class ESubState: ui8 { - AllocateTxId = 0, - Proposed, - Subscribed, - }; - - TString DstPathName; - TPathId DstPathId; - Ydb::Table::CreateTableRequest Scheme; - - EState State = EState::GetScheme; - ESubState SubState = ESubState::AllocateTxId; - TTxId WaitTxId = InvalidTxId; - int NextIndexIdx = 0; - TString Issue; - - TItem() = default; - - explicit TItem(const TString& dstPathName) - : DstPathName(dstPathName) - { - } - - explicit TItem(const TString& dstPathName, const TPathId& dstPathId) - : DstPathName(dstPathName) - , DstPathId(dstPathId) - { - } - - TString ToString(ui32 idx) const; - - static bool IsDone(const TItem& item); - }; - - ui64 Id; - TString Uid; - EKind Kind; - Ydb::Import::ImportFromS3Settings Settings; - TPathId DomainPathId; - TMaybe<TString> UserSID; - - EState State = EState::Invalid; - TString Issue; - TVector<TItem> Items; - - TSet<TActorId> Subscribers; - - explicit TImportInfo( - const ui64 id, - const TString& uid, - const EKind kind, - const Ydb::Import::ImportFromS3Settings& settings, - const TPathId domainPathId) - : Id(id) - , Uid(uid) - , Kind(kind) - , Settings(settings) - , DomainPathId(domainPathId) - { - } - - TString ToString() const; - - bool IsFinished() const; - - void AddNotifySubscriber(const TActorId& actorId); - -}; // TImportInfo -// } // NImport - + return serialized; + } + +}; // TExportInfo +// } // NExport + +// namespace NImport { +struct TImportInfo: public TSimpleRefCount<TImportInfo> { + using TPtr = TIntrusivePtr<TImportInfo>; + + enum class EState: ui8 { + Invalid = 0, + Waiting, + GetScheme, + CreateTable, + Transferring, + BuildIndexes, + Done = 240, + Cancellation = 250, + Cancelled = 251, + }; + + enum class EKind: ui8 { + S3 = 0, + }; + + struct TItem { + enum class ESubState: ui8 { + AllocateTxId = 0, + Proposed, + Subscribed, + }; + + TString DstPathName; + TPathId DstPathId; + Ydb::Table::CreateTableRequest Scheme; + + EState State = EState::GetScheme; + ESubState SubState = ESubState::AllocateTxId; + TTxId WaitTxId = InvalidTxId; + int NextIndexIdx = 0; + TString Issue; + + TItem() = default; + + explicit TItem(const TString& dstPathName) + : DstPathName(dstPathName) + { + } + + explicit TItem(const TString& dstPathName, const TPathId& dstPathId) + : DstPathName(dstPathName) + , DstPathId(dstPathId) + { + } + + TString ToString(ui32 idx) const; + + static bool IsDone(const TItem& item); + }; + + ui64 Id; + TString Uid; + EKind Kind; + Ydb::Import::ImportFromS3Settings Settings; + TPathId DomainPathId; + TMaybe<TString> UserSID; + + EState State = EState::Invalid; + TString Issue; + TVector<TItem> Items; + + TSet<TActorId> Subscribers; + + explicit TImportInfo( + const ui64 id, + const TString& uid, + const EKind kind, + const Ydb::Import::ImportFromS3Settings& settings, + const TPathId domainPathId) + : Id(id) + , Uid(uid) + , Kind(kind) + , Settings(settings) + , DomainPathId(domainPathId) + { + } + + TString ToString() const; + + bool IsFinished() const; + + void AddNotifySubscriber(const TActorId& actorId); + +}; // TImportInfo +// } // NImport + class TBillingStats { public: TBillingStats() = default; @@ -2657,8 +2657,8 @@ bool ValidateTtlSettingsChange( } } - -template <> + +template <> inline void Out<NKikimr::NSchemeShard::TIndexBuildInfo::TShardStatus> (IOutputStream& o, const NKikimr::NSchemeShard::TIndexBuildInfo::TShardStatus& info) { diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 2b0c1c6e15e..7535ee673c8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -8,7 +8,7 @@ namespace NKikimr { namespace NSchemeShard { -static constexpr ui64 MaxPQStorage = Max<ui64>() / 2; +static constexpr ui64 MaxPQStorage = Max<ui64>() / 2; TPath::TChecker::TChecker(const TPath &path) : Path(path) @@ -263,26 +263,26 @@ const TPath::TChecker& TPath::TChecker::IsUnderTheSameOperation(TTxId txId, TPat return *this; } -const TPath::TChecker& TPath::TChecker::NotUnderTheSameOperation(TTxId txId, TPath::TChecker::EStatus status) const { - if (Failed) { - return *this; - } - - TTxId activeTxId = Path.ActiveOperation(); - if (activeTxId != txId) { - return *this; - } - - Failed = true; - Status = status; - Explain << "path is under the same operation" - << ", pathId: " << Path.Base()->PathId +const TPath::TChecker& TPath::TChecker::NotUnderTheSameOperation(TTxId txId, TPath::TChecker::EStatus status) const { + if (Failed) { + return *this; + } + + TTxId activeTxId = Path.ActiveOperation(); + if (activeTxId != txId) { + return *this; + } + + Failed = true; + Status = status; + Explain << "path is under the same operation" + << ", pathId: " << Path.Base()->PathId << ", path type: " << NKikimrSchemeOp::EPathType_Name(Path.Base()->PathType) << ", path state: " << NKikimrSchemeOp::EPathState_Name(Path.Base()->PathState) - << ", txId: " << txId; - return *this; -} - + << ", txId: " << txId; + return *this; +} + const TPath::TChecker& TPath::TChecker::NoOlapStore(TPath::TChecker::EStatus status) const { if (Failed) { return *this; @@ -366,23 +366,23 @@ const TPath::TChecker& TPath::TChecker::IsSequence(TPath::TChecker::EStatus stat return *this; } -const TPath::TChecker& TPath::TChecker::IsReplication(TPath::TChecker::EStatus status) const { - if (Failed) { - return *this; - } - - if (Path.Base()->IsReplication()) { - return *this; - } - - Failed = true; - Status = status; - Explain << "path is not a replication" - << ", pathId: " << Path.Base()->PathId - << ", path type: " << NKikimrSchemeOp::EPathType_Name(Path.Base()->PathType); - return *this; -} - +const TPath::TChecker& TPath::TChecker::IsReplication(TPath::TChecker::EStatus status) const { + if (Failed) { + return *this; + } + + if (Path.Base()->IsReplication()) { + return *this; + } + + Failed = true; + Status = status; + Explain << "path is not a replication" + << ", pathId: " << Path.Base()->PathId + << ", path type: " << NKikimrSchemeOp::EPathType_Name(Path.Base()->PathType); + return *this; +} + const TPath::TChecker& TPath::TChecker::IsCommonSensePath(TPath::TChecker::EStatus status) const { if (Failed) { return *this; @@ -418,23 +418,23 @@ const TPath::TChecker& TPath::TChecker::IsInsideTableIndexPath(TPath::TChecker:: return *this; } -const TPath::TChecker& TPath::TChecker::IsInsideCdcStreamPath(TPath::TChecker::EStatus status) const { - if (Failed) { - return *this; - } - - if (Path.IsInsideCdcStreamPath()) { - return *this; - } - - Failed = true; - Status = status; - Explain << "path doesn't go through directories towards cdc stream" - << ", it might be a cdc stream or private topic" - << ", pathId " << Path.Base()->PathId; - return *this; -} - +const TPath::TChecker& TPath::TChecker::IsInsideCdcStreamPath(TPath::TChecker::EStatus status) const { + if (Failed) { + return *this; + } + + if (Path.IsInsideCdcStreamPath()) { + return *this; + } + + Failed = true; + Status = status; + Explain << "path doesn't go through directories towards cdc stream" + << ", it might be a cdc stream or private topic" + << ", pathId " << Path.Base()->PathId; + return *this; +} + const TPath::TChecker& TPath::TChecker::IsTable(TPath::TChecker::EStatus status) const { if (Failed) { return *this; @@ -524,7 +524,7 @@ const TPath::TChecker& TPath::TChecker::IsKesus(TPath::TChecker::EStatus status) return *this; } -const TPath::TChecker& TPath::TChecker::IsPQGroup(TPath::TChecker::EStatus status) const { +const TPath::TChecker& TPath::TChecker::IsPQGroup(TPath::TChecker::EStatus status) const { if (Failed) { return *this; } @@ -628,23 +628,23 @@ const TPath::TChecker& TPath::TChecker::IsTableIndex(TPath::TChecker::EStatus st return *this; } -const TPath::TChecker& TPath::TChecker::IsCdcStream(TPath::TChecker::EStatus status) const { - if (Failed) { - return *this; - } - - if (Path.Base()->IsCdcStream()) { - return *this; - } - - Failed = true; - Status = status; - Explain << "path is not a cdc stream" - << ", pathId: " << Path.Base()->PathId +const TPath::TChecker& TPath::TChecker::IsCdcStream(TPath::TChecker::EStatus status) const { + if (Failed) { + return *this; + } + + if (Path.Base()->IsCdcStream()) { + return *this; + } + + Failed = true; + Status = status; + Explain << "path is not a cdc stream" + << ", pathId: " << Path.Base()->PathId << ", path type: " << NKikimrSchemeOp::EPathType_Name(Path.Base()->PathType); - return *this; -} - + return *this; +} + const TPath::TChecker& TPath::TChecker::IsLikeDirectory(TPath::TChecker::EStatus status) const { if (Failed) { return *this; @@ -975,37 +975,37 @@ const TPath::TChecker& TPath::TChecker::NotDeleted(TPath::TChecker::EStatus stat return *this; } -const TPath::TChecker& TPath::TChecker::IsValidACL(const TString& acl, EStatus status) const { - if (Failed) { - return *this; - } - - if (acl.empty()) { - return *this; - } - - auto secObj = MakeHolder<NACLib::TACL>(); - if (Path.IsResolved()) { - secObj.Reset(new NACLib::TACL(Path.Base()->ACL)); - } - - NACLib::TDiffACL diffACL(acl); - secObj->ApplyDiff(diffACL); - const ui64 bytesSize = secObj->SerializeAsString().size(); - - if (bytesSize <= Path.DomainInfo()->GetSchemeLimits().MaxAclBytesSize) { - return *this; - } - - Failed = true; - Status = status; - Explain << "path's ACL is too long" - << ", path: " << Path.PathString() - << ", calculated ACL size: " << bytesSize - << ". Limit: " << Path.DomainInfo()->GetSchemeLimits().MaxAclBytesSize; - return *this; -} - +const TPath::TChecker& TPath::TChecker::IsValidACL(const TString& acl, EStatus status) const { + if (Failed) { + return *this; + } + + if (acl.empty()) { + return *this; + } + + auto secObj = MakeHolder<NACLib::TACL>(); + if (Path.IsResolved()) { + secObj.Reset(new NACLib::TACL(Path.Base()->ACL)); + } + + NACLib::TDiffACL diffACL(acl); + secObj->ApplyDiff(diffACL); + const ui64 bytesSize = secObj->SerializeAsString().size(); + + if (bytesSize <= Path.DomainInfo()->GetSchemeLimits().MaxAclBytesSize) { + return *this; + } + + Failed = true; + Status = status; + Explain << "path's ACL is too long" + << ", path: " << Path.PathString() + << ", calculated ACL size: " << bytesSize + << ". Limit: " << Path.DomainInfo()->GetSchemeLimits().MaxAclBytesSize; + return *this; +} + TPath::TPath(TSchemeShard* ss) : SS(ss) { @@ -1328,12 +1328,12 @@ bool TPath::IsUnderOperation() const { bool result = Base()->PathState != NKikimrSchemeOp::EPathState::EPathStateNoChanges; if (result) { - ui32 summ = (ui32)IsUnderCreating() - + (ui32)IsUnderAltering() - + (ui32)IsUnderCopying() - + (ui32)IsUnderBackuping() - + (ui32)IsUnderRestoring() - + (ui32)IsUnderDeleting() + ui32 summ = (ui32)IsUnderCreating() + + (ui32)IsUnderAltering() + + (ui32)IsUnderCopying() + + (ui32)IsUnderBackuping() + + (ui32)IsUnderRestoring() + + (ui32)IsUnderDeleting() + (ui32)IsUnderDomainUpgrade() + (ui32)IsUnderMoving(); Y_VERIFY_S(summ == 1, @@ -1405,12 +1405,12 @@ bool TPath::IsUnderBackuping() const { return Base()->PathState == NKikimrSchemeOp::EPathState::EPathStateBackup; } -bool TPath::IsUnderRestoring() const { - Y_VERIFY(IsResolved()); - +bool TPath::IsUnderRestoring() const { + Y_VERIFY(IsResolved()); + return Base()->PathState == NKikimrSchemeOp::EPathState::EPathStateRestore; -} - +} + bool TPath::IsUnderDeleting() const { Y_VERIFY(IsResolved()); @@ -1508,46 +1508,46 @@ bool TPath::IsInsideTableIndexPath() const { return true; } -bool TPath::IsInsideCdcStreamPath() const { - Y_VERIFY(IsResolved()); - - // expected /<root>/.../<table>/<cdc_stream>/<private_topic> - if (Depth() < 3) { - return false; - } - - auto item = Elements.rbegin(); - - //skip private_topic - if ((*item)->IsPQGroup()) { - ++item; - } - - if (!(*item)->IsCdcStream()) { - return false; - } - - ++item; - if (!(*item)->IsTable()) { - return false; - } - - ++item; - for (; item != Elements.rend(); ++item) { - if (!(*item)->IsDirectory() && !(*item)->IsSubDomainRoot()) { - return false; - } - } - - return true; -} - -bool TPath::IsTableIndex() const { - Y_VERIFY(IsResolved()); - - return Base()->IsTableIndex(); -} - +bool TPath::IsInsideCdcStreamPath() const { + Y_VERIFY(IsResolved()); + + // expected /<root>/.../<table>/<cdc_stream>/<private_topic> + if (Depth() < 3) { + return false; + } + + auto item = Elements.rbegin(); + + //skip private_topic + if ((*item)->IsPQGroup()) { + ++item; + } + + if (!(*item)->IsCdcStream()) { + return false; + } + + ++item; + if (!(*item)->IsTable()) { + return false; + } + + ++item; + for (; item != Elements.rend(); ++item) { + if (!(*item)->IsDirectory() && !(*item)->IsSubDomainRoot()) { + return false; + } + } + + return true; +} + +bool TPath::IsTableIndex() const { + Y_VERIFY(IsResolved()); + + return Base()->IsTableIndex(); +} + bool TPath::IsBackupTable() const { Y_VERIFY(IsResolved()); @@ -1560,24 +1560,24 @@ bool TPath::IsBackupTable() const { return tableInfo->IsBackup; } -bool TPath::IsCdcStream() const { - Y_VERIFY(IsResolved()); - - return Base()->IsCdcStream(); -} - +bool TPath::IsCdcStream() const { + Y_VERIFY(IsResolved()); + + return Base()->IsCdcStream(); +} + bool TPath::IsSequence() const { Y_VERIFY(IsResolved()); return Base()->IsSequence(); } -bool TPath::IsReplication() const { - Y_VERIFY(IsResolved()); - - return Base()->IsReplication(); -} - +bool TPath::IsReplication() const { + Y_VERIFY(IsResolved()); + + return Base()->IsReplication(); +} + ui32 TPath::Depth() const { return NameParts.size(); } @@ -1616,7 +1616,7 @@ bool TPath::IsValidLeafName(TString& explain) const { return false; } - if (AppData()->FeatureFlags.GetEnableSystemViews() && leaf == NSysView::SysPathName) { + if (AppData()->FeatureFlags.GetEnableSystemViews() && leaf == NSysView::SysPathName) { explain += TStringBuilder() << "path part '" << NSysView::SysPathName << "' is reserved by the system"; return false; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index aa1f8b5992a..fb33c1cdd98 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -55,22 +55,22 @@ public: const TChecker& IsOlapStore(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsOlapTable(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsSequence(EStatus status = EStatus::StatusNameConflict) const; - const TChecker& IsReplication(EStatus status = EStatus::StatusNameConflict) const; + const TChecker& IsReplication(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsCommonSensePath(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsInsideTableIndexPath(EStatus status = EStatus::StatusNameConflict) const; - const TChecker& IsInsideCdcStreamPath(EStatus status = EStatus::StatusNameConflict) const; + const TChecker& IsInsideCdcStreamPath(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsTable(EStatus status = EStatus::StatusNameConflict) const; const TChecker& NotBackupTable(EStatus status = NKikimrScheme::StatusSchemeError) const; const TChecker& IsBlockStoreVolume(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsFileStore(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsKesus(EStatus status = EStatus::StatusNameConflict) const; - const TChecker& IsPQGroup(EStatus status = EStatus::StatusNameConflict) const; + const TChecker& IsPQGroup(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsSubDomain(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsExternalSubDomain(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsAtLocalSchemeShard(EStatus status = EStatus::StatusRedirectDomain) const; const TChecker& IsSolomon(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsTableIndex(EStatus status = EStatus::StatusNameConflict) const; - const TChecker& IsCdcStream(EStatus status = EStatus::StatusNameConflict) const; + const TChecker& IsCdcStream(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsLikeDirectory(EStatus status = EStatus::StatusPathIsNotDirectory) const; const TChecker& IsDirectory(EStatus status = EStatus::StatusPathIsNotDirectory) const; const TChecker& IsTheSameDomain(const TPath& another, EStatus status = EStatus::StatusInvalidParameter) const; @@ -83,7 +83,7 @@ public: const TChecker& ShardsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& PathShardsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& NotChildren(EStatus status = EStatus::StatusInvalidParameter) const; - const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const; + const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const; const TChecker& PQPartitionsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& PQReservedStorageLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; }; @@ -135,7 +135,7 @@ public: bool IsUnderDomainUpgrade() const; bool IsUnderCopying() const; bool IsUnderBackuping() const; - bool IsUnderRestoring() const; + bool IsUnderRestoring() const; bool IsUnderDeleting() const; bool IsUnderMoving() const; TPath& RiseUntilOlapStore(); @@ -143,12 +143,12 @@ public: bool IsCommonSensePath() const; bool AtLocalSchemeShardPath() const; bool IsInsideTableIndexPath() const; - bool IsInsideCdcStreamPath() const; - bool IsTableIndex() const; + bool IsInsideCdcStreamPath() const; + bool IsTableIndex() const; bool IsBackupTable() const; - bool IsCdcStream() const; + bool IsCdcStream() const; bool IsSequence() const; - bool IsReplication() const; + bool IsReplication() const; ui32 Depth() const; ui64 Shards() const; const TString& LeafName() const; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 3c631ac0158..3789d3af1b8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -1,36 +1,36 @@ -#include "schemeshard_path_describer.h" - +#include "schemeshard_path_describer.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> - -#include <util/stream/format.h> - -namespace NKikimr { + +#include <util/stream/format.h> + +namespace NKikimr { namespace NSchemeShard { - -static void FillTableStats(NKikimrTableStats::TTableStats* stats, const TTableInfo::TPartitionStats& tableStats) { - stats->SetRowCount(tableStats.RowCount); - stats->SetDataSize(tableStats.DataSize); - stats->SetIndexSize(tableStats.IndexSize); - stats->SetLastAccessTime(tableStats.LastAccessTime.MilliSeconds()); - stats->SetLastUpdateTime(tableStats.LastUpdateTime.MilliSeconds()); - stats->SetImmediateTxCompleted(tableStats.ImmediateTxCompleted); - stats->SetPlannedTxCompleted(tableStats.PlannedTxCompleted); - stats->SetTxRejectedByOverload(tableStats.TxRejectedByOverload); - stats->SetTxRejectedBySpace(tableStats.TxRejectedBySpace); - stats->SetTxCompleteLagMsec(tableStats.TxCompleteLag.MilliSeconds()); - stats->SetInFlightTxCount(tableStats.InFlightTxCount); - - stats->SetRowUpdates(tableStats.RowUpdates); - stats->SetRowDeletes(tableStats.RowDeletes); - stats->SetRowReads(tableStats.RowReads); - stats->SetRangeReads(tableStats.RangeReads); - stats->SetRangeReadRows(tableStats.RangeReadRows); - - stats->SetPartCount(tableStats.PartCount); -} - + +static void FillTableStats(NKikimrTableStats::TTableStats* stats, const TTableInfo::TPartitionStats& tableStats) { + stats->SetRowCount(tableStats.RowCount); + stats->SetDataSize(tableStats.DataSize); + stats->SetIndexSize(tableStats.IndexSize); + stats->SetLastAccessTime(tableStats.LastAccessTime.MilliSeconds()); + stats->SetLastUpdateTime(tableStats.LastUpdateTime.MilliSeconds()); + stats->SetImmediateTxCompleted(tableStats.ImmediateTxCompleted); + stats->SetPlannedTxCompleted(tableStats.PlannedTxCompleted); + stats->SetTxRejectedByOverload(tableStats.TxRejectedByOverload); + stats->SetTxRejectedBySpace(tableStats.TxRejectedBySpace); + stats->SetTxCompleteLagMsec(tableStats.TxCompleteLag.MilliSeconds()); + stats->SetInFlightTxCount(tableStats.InFlightTxCount); + + stats->SetRowUpdates(tableStats.RowUpdates); + stats->SetRowDeletes(tableStats.RowDeletes); + stats->SetRowReads(tableStats.RowReads); + stats->SetRangeReads(tableStats.RangeReads); + stats->SetRangeReadRows(tableStats.RangeReadRows); + + stats->SetPartCount(tableStats.PartCount); +} + static void FillTableMetrics(NKikimrTabletBase::TMetrics* metrics, const TTableInfo::TPartitionStats& tableStats) { metrics->SetCPU(tableStats.GetCurrentRawCpuUsage()); metrics->SetMemory(tableStats.Memory); @@ -50,16 +50,16 @@ void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathEleme } void TPathDescriber::FillChildDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl) { - bool createFinished = pathEl->IsCreateFinished(); - - descr->SetName(pathEl->Name); + bool createFinished = pathEl->IsCreateFinished(); + + descr->SetName(pathEl->Name); descr->SetSchemeshardId(pathEl->PathId.OwnerId); descr->SetPathId(pathEl->PathId.LocalPathId); descr->SetParentPathId(pathEl->ParentPathId.LocalPathId); //???? ParnetPathOwnerId - descr->SetPathType(pathEl->PathType); + descr->SetPathType(pathEl->PathType); descr->SetOwner(pathEl->Owner); descr->SetPathState(pathEl->PathState); // ???? can't be consistent KIKIMR-8861 @@ -69,96 +69,96 @@ void TPathDescriber::FillChildDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElem if (createFinished) { descr->SetCreateStep(ui64(pathEl->StepCreated)); } - + if (pathEl->PathType == NKikimrSchemeOp::EPathTypePersQueueGroup) { - auto it = Self->PersQueueGroups.FindPtr(pathEl->PathId); - Y_VERIFY(it, "PersQueueGroup is not found"); - - TPersQueueGroupInfo::TPtr pqGroupInfo = *it; - if (pqGroupInfo->HasBalancer()) { + auto it = Self->PersQueueGroups.FindPtr(pathEl->PathId); + Y_VERIFY(it, "PersQueueGroup is not found"); + + TPersQueueGroupInfo::TPtr pqGroupInfo = *it; + if (pqGroupInfo->HasBalancer()) { descr->SetBalancerTabletID(ui64(pqGroupInfo->BalancerTabletID)); - } + } } else { descr->SetACL(pathEl->ACL); // YDBOPS-1328 - } -} - -TPathElement::EPathSubType TPathDescriber::CalcPathSubType(const TPath& path) { - if (!path.IsResolved()) { - return TPathElement::EPathSubType::EPathSubTypeEmpty; - } - - if (path.IsCommonSensePath()) { - return TPathElement::EPathSubType::EPathSubTypeEmpty; - } - - const auto parentPath = path.Parent(); - Y_VERIFY(parentPath.IsResolved()); - - if (parentPath.IsTableIndex()) { - const auto& pathId = parentPath.Base()->PathId; - Y_VERIFY(Self->Indexes.contains(pathId)); - auto indexInfo = Self->Indexes.at(pathId); - - switch (indexInfo->Type) { - case NKikimrSchemeOp::EIndexTypeGlobalAsync: - return TPathElement::EPathSubType::EPathSubTypeAsyncIndexImplTable; - default: - return TPathElement::EPathSubType::EPathSubTypeSyncIndexImplTable; - } - } else if (parentPath.IsCdcStream()) { - return TPathElement::EPathSubType::EPathSubTypeStreamImpl; - } - - return TPathElement::EPathSubType::EPathSubTypeEmpty; -} - + } +} + +TPathElement::EPathSubType TPathDescriber::CalcPathSubType(const TPath& path) { + if (!path.IsResolved()) { + return TPathElement::EPathSubType::EPathSubTypeEmpty; + } + + if (path.IsCommonSensePath()) { + return TPathElement::EPathSubType::EPathSubTypeEmpty; + } + + const auto parentPath = path.Parent(); + Y_VERIFY(parentPath.IsResolved()); + + if (parentPath.IsTableIndex()) { + const auto& pathId = parentPath.Base()->PathId; + Y_VERIFY(Self->Indexes.contains(pathId)); + auto indexInfo = Self->Indexes.at(pathId); + + switch (indexInfo->Type) { + case NKikimrSchemeOp::EIndexTypeGlobalAsync: + return TPathElement::EPathSubType::EPathSubTypeAsyncIndexImplTable; + default: + return TPathElement::EPathSubType::EPathSubTypeSyncIndexImplTable; + } + } else if (parentPath.IsCdcStream()) { + return TPathElement::EPathSubType::EPathSubTypeStreamImpl; + } + + return TPathElement::EPathSubType::EPathSubTypeEmpty; +} + void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, const TPath& path) { - FillPathDescr(descr, path.Base(), CalcPathSubType(path)); -} - + FillPathDescr(descr, path.Base(), CalcPathSubType(path)); +} + void TPathDescriber::BuildEffectiveACL(NKikimrSchemeOp::TDirEntry* descr, const TPath& path) { descr->SetEffectiveACL(path.GetEffectiveACL()); -} - -void TPathDescriber::FillLastExistedPrefixDescr(const TPath& path) { +} + +void TPathDescriber::FillLastExistedPrefixDescr(const TPath& path) { if (path.IsEmpty()) { - return; - } + return; + } Y_VERIFY(path.IsResolved()); Y_VERIFY(!path.IsDeleted()); - + Result->Record.SetLastExistedPrefixPathId(path.Base()->PathId.LocalPathId); Result->Record.SetLastExistedPrefixPath(path.PathString()); - + Y_VERIFY_S(Self->PathsById.contains(path.Base()->PathId), "Unknown pathId " << path.Base()->PathId); - auto descr = Result->Record.MutableLastExistedPrefixDescription()->MutableSelf(); - FillPathDescr(descr, path); + auto descr = Result->Record.MutableLastExistedPrefixDescription()->MutableSelf(); + FillPathDescr(descr, path); BuildEffectiveACL(descr, path); -} - +} + void TPathDescriber::DescribeChildren(const TPath& path) { auto pathEl = path.Base(); if (!Params.GetOptions().GetReturnChildren()) { - return; - } - + return; + } + if (pathEl->IsTable()) { return; } - if (pathEl->PreSerializedChildrenListing.empty()) { + if (pathEl->PreSerializedChildrenListing.empty()) { NKikimrScheme::TEvDescribeSchemeResult preSerializedResult; - auto pathDescription = preSerializedResult.MutablePathDescription(); + auto pathDescription = preSerializedResult.MutablePathDescription(); pathDescription->MutableChildren()->Reserve(pathEl->GetAliveChildren()); - for (const auto& child : pathEl->GetChildren()) { - TPathId childId = child.second; - TPathElement::TPtr childEl = *Self->PathsById.FindPtr(childId); + for (const auto& child : pathEl->GetChildren()) { + TPathId childId = child.second; + TPathElement::TPtr childEl = *Self->PathsById.FindPtr(childId); if (childEl->Dropped() || childEl->IsMigrated()) { - continue; - } - auto entry = pathDescription->AddChildren(); + continue; + } + auto entry = pathDescription->AddChildren(); if (pathEl->IsTableIndex()) { // we put version of child here in addition @@ -166,187 +166,187 @@ void TPathDescriber::DescribeChildren(const TPath& path) { FillPathDescr(entry, childEl, CalcPathSubType(childPath)); const auto version = Self->GetPathVersion(childPath); entry->MutableVersion()->CopyFrom(version); - } else if (pathEl->IsCdcStream()) { - auto childPath = path.Child(child.first); - FillPathDescr(entry, childEl, CalcPathSubType(childPath)); + } else if (pathEl->IsCdcStream()) { + auto childPath = path.Child(child.first); + FillPathDescr(entry, childEl, CalcPathSubType(childPath)); } else { FillChildDescr(entry, childEl); } - } + } Y_PROTOBUF_SUPPRESS_NODISCARD preSerializedResult.SerializeToString(&pathEl->PreSerializedChildrenListing); - } - Result->PreSerializedData += pathEl->PreSerializedChildrenListing; + } + Result->PreSerializedData += pathEl->PreSerializedChildrenListing; + + if (!pathEl->IsCreateFinished()) { + pathEl->PreSerializedChildrenListing.clear(); + } +} - if (!pathEl->IsCreateFinished()) { - pathEl->PreSerializedChildrenListing.clear(); - } -} - void TPathDescriber::DescribeDir(const TPath& path) { DescribeChildren(path); } -void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl) { - const NScheme::TTypeRegistry* typeRegistry = AppData(ctx)->TypeRegistry; - const TTableInfo::TPtr tableInfo = *Self->Tables.FindPtr(pathId); - auto pathDescription = Result->Record.MutablePathDescription(); - auto entry = pathDescription->MutableTable(); - - bool returnConfig = Params.GetReturnPartitionConfig(); - bool returnPartitioning = Params.GetReturnPartitioningInfo(); - bool returnPartitionStats = Params.GetOptions().GetReturnPartitionStats(); - bool returnBackupInfo = Params.GetBackupInfo(); - bool returnBoundaries = false; - if (Params.HasOptions()) { - returnConfig = Params.GetOptions().GetReturnPartitionConfig(); - returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo(); - returnBackupInfo = Params.GetOptions().GetBackupInfo(); - returnBoundaries = Params.GetOptions().GetReturnBoundaries(); - } - - Self->DescribeTable(tableInfo, typeRegistry, returnConfig, returnBoundaries, entry); - entry->SetName(pathEl->Name); - - if (returnPartitioning) { - // partitions - if (tableInfo->PreSerializedPathDescription.empty()) { +void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl) { + const NScheme::TTypeRegistry* typeRegistry = AppData(ctx)->TypeRegistry; + const TTableInfo::TPtr tableInfo = *Self->Tables.FindPtr(pathId); + auto pathDescription = Result->Record.MutablePathDescription(); + auto entry = pathDescription->MutableTable(); + + bool returnConfig = Params.GetReturnPartitionConfig(); + bool returnPartitioning = Params.GetReturnPartitioningInfo(); + bool returnPartitionStats = Params.GetOptions().GetReturnPartitionStats(); + bool returnBackupInfo = Params.GetBackupInfo(); + bool returnBoundaries = false; + if (Params.HasOptions()) { + returnConfig = Params.GetOptions().GetReturnPartitionConfig(); + returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo(); + returnBackupInfo = Params.GetOptions().GetBackupInfo(); + returnBoundaries = Params.GetOptions().GetReturnBoundaries(); + } + + Self->DescribeTable(tableInfo, typeRegistry, returnConfig, returnBoundaries, entry); + entry->SetName(pathEl->Name); + + if (returnPartitioning) { + // partitions + if (tableInfo->PreSerializedPathDescription.empty()) { NKikimrScheme::TEvDescribeSchemeResult preSerializedResult; NKikimrSchemeOp::TPathDescription& pathDescription = *preSerializedResult.MutablePathDescription(); - pathDescription.MutableTablePartitions()->Reserve(tableInfo->GetPartitions().size()); - for (auto& p : tableInfo->GetPartitions()) { - auto part = pathDescription.AddTablePartitions(); + pathDescription.MutableTablePartitions()->Reserve(tableInfo->GetPartitions().size()); + for (auto& p : tableInfo->GetPartitions()) { + auto part = pathDescription.AddTablePartitions(); auto datashardIdx = p.ShardIdx; auto datashardTabletId = Self->ShardInfos[datashardIdx].TabletID; - // Currently we only support uniform partitioning where each range is [start, end) - // +inf as the end of the last range is represented by empty TCell vector + // Currently we only support uniform partitioning where each range is [start, end) + // +inf as the end of the last range is represented by empty TCell vector part->SetDatashardId(ui64(datashardTabletId)); - part->SetIsPoint(false); - part->SetIsInclusive(false); - part->SetEndOfRangeKeyPrefix(p.EndOfRange); - } + part->SetIsPoint(false); + part->SetIsInclusive(false); + part->SetEndOfRangeKeyPrefix(p.EndOfRange); + } Y_PROTOBUF_SUPPRESS_NODISCARD preSerializedResult.SerializeToString(&tableInfo->PreSerializedPathDescription); - } - Result->PreSerializedData += tableInfo->PreSerializedPathDescription; - if (!pathEl->IsCreateFinished()) { - tableInfo->PreSerializedPathDescription.clear(); // KIKIMR-4337 - } - } - - const auto& tableStats(tableInfo->GetStats().Aggregated); - - { - auto* stats = Result->Record.MutablePathDescription()->MutableTableStats(); - FillTableStats(stats, tableStats); - } - - { - auto* metrics = Result->Record.MutablePathDescription()->MutableTabletMetrics(); + } + Result->PreSerializedData += tableInfo->PreSerializedPathDescription; + if (!pathEl->IsCreateFinished()) { + tableInfo->PreSerializedPathDescription.clear(); // KIKIMR-4337 + } + } + + const auto& tableStats(tableInfo->GetStats().Aggregated); + + { + auto* stats = Result->Record.MutablePathDescription()->MutableTableStats(); + FillTableStats(stats, tableStats); + } + + { + auto* metrics = Result->Record.MutablePathDescription()->MutableTabletMetrics(); FillTableMetrics(metrics, tableStats); - } - - if (returnPartitionStats) { + } + + if (returnPartitionStats) { NKikimrSchemeOp::TPathDescription& pathDescription = *Result->Record.MutablePathDescription(); - pathDescription.MutableTablePartitionStats()->Reserve(tableInfo->GetPartitions().size()); - for (auto& p : tableInfo->GetPartitions()) { - const auto* stats = tableInfo->GetStats().PartitionStats.FindPtr(p.ShardIdx); - Y_VERIFY(stats); - auto pbStats = pathDescription.AddTablePartitionStats(); - FillTableStats(pbStats, *stats); + pathDescription.MutableTablePartitionStats()->Reserve(tableInfo->GetPartitions().size()); + for (auto& p : tableInfo->GetPartitions()) { + const auto* stats = tableInfo->GetStats().PartitionStats.FindPtr(p.ShardIdx); + Y_VERIFY(stats); + auto pbStats = pathDescription.AddTablePartitionStats(); + FillTableStats(pbStats, *stats); auto pbMetrics = pathDescription.AddTablePartitionMetrics(); FillTableMetrics(pbMetrics, *stats); - } - } - - if (returnBackupInfo) { - /* Find active backup */ - for (const auto& pair: Self->TxInFlight) { - const auto& txId = pair.first.GetTxId(); - const auto& txState = pair.second; - if (txState.TargetPathId != pathId) - continue; - if (txState.TxType != TTxState::TxBackup) - continue; - - ui32 notCompleteYet = txState.ShardsInProgress.size(); - ui32 total = txState.Shards.size(); - - auto progress = pathDescription->MutableBackupProgress(); - progress->SetNotCompleteYet(notCompleteYet); - progress->SetTotal(total); - progress->SetErrorCount(CountIf(txState.ShardStatuses, [](const auto& kv) { - return !kv.second.Success; - })); - progress->SetStartTime(txState.StartTime.Seconds()); - progress->MutableYTSettings()->CopyFrom( - tableInfo->BackupSettings.GetYTSettings()); - progress->SetDataTotalSize(txState.DataTotalSize); + } + } + + if (returnBackupInfo) { + /* Find active backup */ + for (const auto& pair: Self->TxInFlight) { + const auto& txId = pair.first.GetTxId(); + const auto& txState = pair.second; + if (txState.TargetPathId != pathId) + continue; + if (txState.TxType != TTxState::TxBackup) + continue; + + ui32 notCompleteYet = txState.ShardsInProgress.size(); + ui32 total = txState.Shards.size(); + + auto progress = pathDescription->MutableBackupProgress(); + progress->SetNotCompleteYet(notCompleteYet); + progress->SetTotal(total); + progress->SetErrorCount(CountIf(txState.ShardStatuses, [](const auto& kv) { + return !kv.second.Success; + })); + progress->SetStartTime(txState.StartTime.Seconds()); + progress->MutableYTSettings()->CopyFrom( + tableInfo->BackupSettings.GetYTSettings()); + progress->SetDataTotalSize(txState.DataTotalSize); progress->SetTxId(ui64(txId)); - + LOG_TRACE(ctx, NKikimrServices::SCHEMESHARD_DESCRIBE, - "Add backup process info"); - break; - } - - /* Get information about last completed backup */ - for (const auto& iter: tableInfo->BackupHistory) { + "Add backup process info"); + break; + } + + /* Get information about last completed backup */ + for (const auto& iter: tableInfo->BackupHistory) { LOG_TRACE(ctx, NKikimrServices::SCHEMESHARD_DESCRIBE, "Add last backup info item to history"); - auto protoResult = pathDescription->AddLastBackupResult(); - const auto& txId = iter.first; - const auto& tInfoResult = iter.second; - protoResult->SetErrorCount( - tInfoResult.TotalShardCount - tInfoResult.SuccessShardCount); - protoResult->SetCompleteTimeStamp( - tInfoResult.CompletionDateTime); - protoResult->SetStartTimeStamp(tInfoResult.StartDateTime); - protoResult->SetDataTotalSize( - tInfoResult.DataTotalSize); + auto protoResult = pathDescription->AddLastBackupResult(); + const auto& txId = iter.first; + const auto& tInfoResult = iter.second; + protoResult->SetErrorCount( + tInfoResult.TotalShardCount - tInfoResult.SuccessShardCount); + protoResult->SetCompleteTimeStamp( + tInfoResult.CompletionDateTime); + protoResult->SetStartTimeStamp(tInfoResult.StartDateTime); + protoResult->SetDataTotalSize( + tInfoResult.DataTotalSize); protoResult->SetTxId(ui64(txId)); - - for (const auto& [shardId, status]: tInfoResult.ShardStatuses) { - if (status.Success) { - continue; - } - - auto shardError = protoResult->AddErrors(); - shardError->SetShardId(ui64(shardId.GetLocalId())); - shardError->SetExplain(status.Error); - } - } - } - - for (const auto& child : pathEl->GetChildren()) { - const auto& childName = child.first; - const auto& childPathId = child.second; - - Y_VERIFY(Self->PathsById.contains(childPathId)); - auto childPath = Self->PathsById.at(childPathId); - - if (childPath->Dropped()) { + + for (const auto& [shardId, status]: tInfoResult.ShardStatuses) { + if (status.Success) { + continue; + } + + auto shardError = protoResult->AddErrors(); + shardError->SetShardId(ui64(shardId.GetLocalId())); + shardError->SetExplain(status.Error); + } + } + } + + for (const auto& child : pathEl->GetChildren()) { + const auto& childName = child.first; + const auto& childPathId = child.second; + + Y_VERIFY(Self->PathsById.contains(childPathId)); + auto childPath = Self->PathsById.at(childPathId); + + if (childPath->Dropped()) { continue; } - switch (childPath->PathType) { + switch (childPath->PathType) { case NKikimrSchemeOp::EPathTypeTableIndex: - Self->DescribeTableIndex(childPathId, childName, *entry->AddTableIndexes()); - break; + Self->DescribeTableIndex(childPathId, childName, *entry->AddTableIndexes()); + break; case NKikimrSchemeOp::EPathTypeCdcStream: - Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams()); - break; + Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams()); + break; case NKikimrSchemeOp::EPathTypeSequence: Self->DescribeSequence(childPathId, childName, *entry->AddSequences()); break; - default: - Y_FAIL_S("Unexpected table's child" - << ": tableId# " << pathId - << ", childId# " << childPathId - << ", childName# " << childName - << ", childType# " << static_cast<ui32>(childPath->PathType)); - } - } -} - + default: + Y_FAIL_S("Unexpected table's child" + << ": tableId# " << pathId + << ", childId# " << childPathId + << ", childName# " << childName + << ", childType# " << static_cast<ui32>(childPath->PathType)); + } + } +} + void TPathDescriber::DescribeOlapStore(TPathId pathId, TPathElement::TPtr pathEl) { const TOlapStoreInfo::TPtr storeInfo = *Self->OlapStores.FindPtr(pathId); Y_VERIFY(storeInfo, "OlapStore not found"); @@ -396,190 +396,190 @@ void TPathDescriber::DescribeOlapTable(TPathId pathId, TPathElement::TPtr pathEl #endif } -void TPathDescriber::DescribePersQueueGroup(TPathId pathId, TPathElement::TPtr pathEl) { - auto it = Self->PersQueueGroups.FindPtr(pathId); - Y_VERIFY(it, "PersQueueGroup is not found"); - TPersQueueGroupInfo::TPtr pqGroupInfo = *it; - - if (pqGroupInfo->PreSerializedPathDescription.empty()) { +void TPathDescriber::DescribePersQueueGroup(TPathId pathId, TPathElement::TPtr pathEl) { + auto it = Self->PersQueueGroups.FindPtr(pathId); + Y_VERIFY(it, "PersQueueGroup is not found"); + TPersQueueGroupInfo::TPtr pqGroupInfo = *it; + + if (pqGroupInfo->PreSerializedPathDescription.empty()) { NKikimrScheme::TEvDescribeSchemeResult preSerializedResult; - auto entry = preSerializedResult.MutablePathDescription()->MutablePersQueueGroup(); - entry->SetName(pathEl->Name); + auto entry = preSerializedResult.MutablePathDescription()->MutablePersQueueGroup(); + entry->SetName(pathEl->Name); entry->SetPathId(pathId.LocalPathId); - entry->SetNextPartitionId(pqGroupInfo->NextPartitionId); - entry->SetTotalGroupCount(pqGroupInfo->TotalGroupCount); - entry->SetPartitionPerTablet(pqGroupInfo->MaxPartsPerTablet); - entry->SetAlterVersion(pqGroupInfo->AlterVersion); + entry->SetNextPartitionId(pqGroupInfo->NextPartitionId); + entry->SetTotalGroupCount(pqGroupInfo->TotalGroupCount); + entry->SetPartitionPerTablet(pqGroupInfo->MaxPartsPerTablet); + entry->SetAlterVersion(pqGroupInfo->AlterVersion); Y_PROTOBUF_SUPPRESS_NODISCARD entry->MutablePQTabletConfig()->ParseFromString(pqGroupInfo->TabletConfig); - - if (pqGroupInfo->HasBalancer()) { + + if (pqGroupInfo->HasBalancer()) { entry->SetBalancerTabletID(ui64(pqGroupInfo->BalancerTabletID)); - } - + } + Y_PROTOBUF_SUPPRESS_NODISCARD preSerializedResult.SerializeToString(&pqGroupInfo->PreSerializedPathDescription); - } - - Y_VERIFY_DEBUG(!pqGroupInfo->PreSerializedPathDescription.empty()); - Result->PreSerializedData += pqGroupInfo->PreSerializedPathDescription; - - bool returnPartitioning = Params.GetReturnPartitioningInfo(); - if (returnPartitioning) { - if (pqGroupInfo->PreSerializedPartitionsDescription.empty()) { + } + + Y_VERIFY_DEBUG(!pqGroupInfo->PreSerializedPathDescription.empty()); + Result->PreSerializedData += pqGroupInfo->PreSerializedPathDescription; + + bool returnPartitioning = Params.GetReturnPartitioningInfo(); + if (returnPartitioning) { + if (pqGroupInfo->PreSerializedPartitionsDescription.empty()) { NKikimrScheme::TEvDescribeSchemeResult preSerializedResult; - auto entry = preSerializedResult.MutablePathDescription()->MutablePersQueueGroup(); - - struct TPartitionDesc { - TTabletId TabletId = InvalidTabletId; - const TPQShardInfo::TPersQueueInfo* Info = nullptr; - }; - - TVector<TPartitionDesc> descriptions; // index is pqId - descriptions.resize(pqGroupInfo->TotalPartitionCount); - - for (const auto& [shardIdx, pqShard] : pqGroupInfo->Shards) { - auto it = Self->ShardInfos.find(shardIdx); - Y_VERIFY_S(it != Self->ShardInfos.end(), "No shard with shardIdx: " << shardIdx); - - for (const auto& pq : pqShard->PQInfos) { - if (pq.AlterVersion <= pqGroupInfo->AlterVersion) { - Y_VERIFY_S(pq.PqId < pqGroupInfo->NextPartitionId, - "Wrong pqId: " << pq.PqId << ", nextPqId: " << pqGroupInfo->NextPartitionId); - descriptions[pq.PqId] = {it->second.TabletID, &pq}; - } - } - } - - for (ui32 pqId = 0; pqId < descriptions.size(); ++pqId) { - const auto& desc = descriptions.at(pqId); - auto& partition = *entry->AddPartitions(); - - Y_VERIFY_S(desc.TabletId, "Unassigned tabetId for partition: " << pqId); - Y_VERIFY_S(desc.Info, "Empty info for partition: " << pqId); - - partition.SetPartitionId(pqId); - partition.SetTabletId(ui64(desc.TabletId)); - if (desc.Info->KeyRange) { - desc.Info->KeyRange->SerializeToProto(*partition.MutableKeyRange()); - } - } - + auto entry = preSerializedResult.MutablePathDescription()->MutablePersQueueGroup(); + + struct TPartitionDesc { + TTabletId TabletId = InvalidTabletId; + const TPQShardInfo::TPersQueueInfo* Info = nullptr; + }; + + TVector<TPartitionDesc> descriptions; // index is pqId + descriptions.resize(pqGroupInfo->TotalPartitionCount); + + for (const auto& [shardIdx, pqShard] : pqGroupInfo->Shards) { + auto it = Self->ShardInfos.find(shardIdx); + Y_VERIFY_S(it != Self->ShardInfos.end(), "No shard with shardIdx: " << shardIdx); + + for (const auto& pq : pqShard->PQInfos) { + if (pq.AlterVersion <= pqGroupInfo->AlterVersion) { + Y_VERIFY_S(pq.PqId < pqGroupInfo->NextPartitionId, + "Wrong pqId: " << pq.PqId << ", nextPqId: " << pqGroupInfo->NextPartitionId); + descriptions[pq.PqId] = {it->second.TabletID, &pq}; + } + } + } + + for (ui32 pqId = 0; pqId < descriptions.size(); ++pqId) { + const auto& desc = descriptions.at(pqId); + auto& partition = *entry->AddPartitions(); + + Y_VERIFY_S(desc.TabletId, "Unassigned tabetId for partition: " << pqId); + Y_VERIFY_S(desc.Info, "Empty info for partition: " << pqId); + + partition.SetPartitionId(pqId); + partition.SetTabletId(ui64(desc.TabletId)); + if (desc.Info->KeyRange) { + desc.Info->KeyRange->SerializeToProto(*partition.MutableKeyRange()); + } + } + Y_PROTOBUF_SUPPRESS_NODISCARD preSerializedResult.SerializeToString(&pqGroupInfo->PreSerializedPartitionsDescription); - } - - Y_VERIFY_DEBUG(!pqGroupInfo->PreSerializedPartitionsDescription.empty()); - Result->PreSerializedData += pqGroupInfo->PreSerializedPartitionsDescription; - } - - Y_VERIFY_DEBUG(!Result->PreSerializedData.empty()); - if (!pathEl->IsCreateFinished()) { - // Don't cache until create finishes (KIKIMR-4337) - pqGroupInfo->PreSerializedPathDescription.clear(); - pqGroupInfo->PreSerializedPartitionsDescription.clear(); - } -} - -void TPathDescriber::DescribeRtmrVolume(TPathId pathId, TPathElement::TPtr pathEl) { - auto it = Self->RtmrVolumes.FindPtr(pathId); - Y_VERIFY(it, "RtmrVolume is not found"); - TRtmrVolumeInfo::TPtr rtmrVolumeInfo = *it; - - auto entry = Result->Record.MutablePathDescription()->MutableRtmrVolumeDescription(); - entry->SetName(pathEl->Name); + } + + Y_VERIFY_DEBUG(!pqGroupInfo->PreSerializedPartitionsDescription.empty()); + Result->PreSerializedData += pqGroupInfo->PreSerializedPartitionsDescription; + } + + Y_VERIFY_DEBUG(!Result->PreSerializedData.empty()); + if (!pathEl->IsCreateFinished()) { + // Don't cache until create finishes (KIKIMR-4337) + pqGroupInfo->PreSerializedPathDescription.clear(); + pqGroupInfo->PreSerializedPartitionsDescription.clear(); + } +} + +void TPathDescriber::DescribeRtmrVolume(TPathId pathId, TPathElement::TPtr pathEl) { + auto it = Self->RtmrVolumes.FindPtr(pathId); + Y_VERIFY(it, "RtmrVolume is not found"); + TRtmrVolumeInfo::TPtr rtmrVolumeInfo = *it; + + auto entry = Result->Record.MutablePathDescription()->MutableRtmrVolumeDescription(); + entry->SetName(pathEl->Name); entry->SetPathId(pathId.LocalPathId); - entry->SetPartitionsCount(rtmrVolumeInfo->Partitions.size()); - - for (const auto& partition: rtmrVolumeInfo->Partitions) { - auto part = entry->AddPartitions(); - part->SetPartitionId((const char*)partition.second->Id.dw, sizeof(TGUID)); - part->SetBusKey(partition.second->BusKey); + entry->SetPartitionsCount(rtmrVolumeInfo->Partitions.size()); + + for (const auto& partition: rtmrVolumeInfo->Partitions) { + auto part = entry->AddPartitions(); + part->SetPartitionId((const char*)partition.second->Id.dw, sizeof(TGUID)); + part->SetBusKey(partition.second->BusKey); part->SetTabletId(ui64(partition.second->TabletId)); - } - - Sort(entry->MutablePartitions()->begin(), - entry->MutablePartitions()->end(), - [](const auto& part1, const auto& part2){ - return part1.GetBusKey() < part2.GetBusKey(); - }); -} - + } + + Sort(entry->MutablePartitions()->begin(), + entry->MutablePartitions()->end(), + [](const auto& part1, const auto& part2){ + return part1.GetBusKey() < part2.GetBusKey(); + }); +} + void TPathDescriber::DescribeTableIndex(const TPath& path) { - Self->DescribeTableIndex(path.Base()->PathId, path.Base()->Name, - *Result->Record.MutablePathDescription()->MutableTableIndex()); + Self->DescribeTableIndex(path.Base()->PathId, path.Base()->Name, + *Result->Record.MutablePathDescription()->MutableTableIndex()); + DescribeChildren(path); +} + +void TPathDescriber::DescribeCdcStream(const TPath& path) { + Self->DescribeCdcStream(path.Base()->PathId, path.Base()->Name, + *Result->Record.MutablePathDescription()->MutableCdcStreamDescription()); DescribeChildren(path); -} - -void TPathDescriber::DescribeCdcStream(const TPath& path) { - Self->DescribeCdcStream(path.Base()->PathId, path.Base()->Name, - *Result->Record.MutablePathDescription()->MutableCdcStreamDescription()); - DescribeChildren(path); -} - -void TPathDescriber::DescribeSolomonVolume(TPathId pathId, TPathElement::TPtr pathEl, bool returnChannelsBinding) { - auto it = Self->SolomonVolumes.FindPtr(pathId); - Y_VERIFY(it, "SolomonVolume is not found"); - TSolomonVolumeInfo::TPtr solomonVolumeInfo = *it; - - auto entry = Result->Record.MutablePathDescription()->MutableSolomonDescription(); - entry->SetName(pathEl->Name); +} + +void TPathDescriber::DescribeSolomonVolume(TPathId pathId, TPathElement::TPtr pathEl, bool returnChannelsBinding) { + auto it = Self->SolomonVolumes.FindPtr(pathId); + Y_VERIFY(it, "SolomonVolume is not found"); + TSolomonVolumeInfo::TPtr solomonVolumeInfo = *it; + + auto entry = Result->Record.MutablePathDescription()->MutableSolomonDescription(); + entry->SetName(pathEl->Name); entry->SetPathId(pathId.LocalPathId); - entry->SetPartitionCount(solomonVolumeInfo->Partitions.size()); - - for (const auto& partition: solomonVolumeInfo->Partitions) { - auto shardId = partition.first; - auto shardInfo = Self->ShardInfos.FindPtr(shardId); - Y_VERIFY(shardInfo); - - auto part = entry->AddPartitions(); - part->SetPartitionId(partition.second->PartitionId); + entry->SetPartitionCount(solomonVolumeInfo->Partitions.size()); + + for (const auto& partition: solomonVolumeInfo->Partitions) { + auto shardId = partition.first; + auto shardInfo = Self->ShardInfos.FindPtr(shardId); + Y_VERIFY(shardInfo); + + auto part = entry->AddPartitions(); + part->SetPartitionId(partition.second->PartitionId); part->SetShardIdx(ui64(shardId.GetLocalId())); - auto tabletId = partition.second->TabletId; - if (tabletId != InvalidTabletId) { + auto tabletId = partition.second->TabletId; + if (tabletId != InvalidTabletId) { part->SetTabletId(ui64(partition.second->TabletId)); - } - - if (returnChannelsBinding) { - for (const auto& channel : shardInfo->BindedChannels) { - part->AddBoundChannels()->CopyFrom(channel); - } - } - } - - Sort(entry->MutablePartitions()->begin(), - entry->MutablePartitions()->end(), - [] (auto& part1, auto& part2) { - return part1.GetPartitionId() < part2.GetPartitionId(); - }); -} - -void TPathDescriber::DescribeUserAttributes(TPathElement::TPtr pathEl) { + } + + if (returnChannelsBinding) { + for (const auto& channel : shardInfo->BindedChannels) { + part->AddBoundChannels()->CopyFrom(channel); + } + } + } + + Sort(entry->MutablePartitions()->begin(), + entry->MutablePartitions()->end(), + [] (auto& part1, auto& part2) { + return part1.GetPartitionId() < part2.GetPartitionId(); + }); +} + +void TPathDescriber::DescribeUserAttributes(TPathElement::TPtr pathEl) { if (!pathEl->UserAttrs->Size() && !pathEl->HasRuntimeAttrs()) { - return; - } - - auto* userAttrs = Result->Record.MutablePathDescription()->MutableUserAttributes(); - for (const auto& item: pathEl->UserAttrs->Attrs) { - auto* attr = userAttrs->Add(); - attr->SetKey(item.first); - attr->SetValue(item.second); - } + return; + } + + auto* userAttrs = Result->Record.MutablePathDescription()->MutableUserAttributes(); + for (const auto& item: pathEl->UserAttrs->Attrs) { + auto* attr = userAttrs->Add(); + attr->SetKey(item.first); + attr->SetValue(item.second); + } pathEl->SerializeRuntimeAttrs(userAttrs); -} - +} + void TPathDescriber::DescribePathVersion(const TPath& path) { const auto& version = Self->GetPathVersion(path); Result->Record.MutablePathDescription()->MutableSelf()->SetPathVersion(version.GetGeneralVersion()); Result->Record.MutablePathDescription()->MutableSelf()->MutableVersion()->CopyFrom(version); -} - +} + void TPathDescriber::DescribeDomain(TPathElement::TPtr pathEl) { TPathId domainId = Self->ResolveDomainId(pathEl); - + TPathElement::TPtr domainEl = Self->PathsById.at(domainId); Y_VERIFY(domainEl); DescribeDomainRoot(domainEl); -} - +} + void TPathDescriber::DescribeRevertedMigrations(TPathElement::TPtr pathEl) { Y_VERIFY(pathEl->IsDomainRoot()); @@ -595,13 +595,13 @@ void TPathDescriber::DescribeRevertedMigrations(TPathElement::TPtr pathEl) { void TPathDescriber::DescribeDomainRoot(TPathElement::TPtr pathEl) { Y_VERIFY(pathEl->IsDomainRoot()); - auto it = Self->SubDomains.FindPtr(pathEl->PathId); + auto it = Self->SubDomains.FindPtr(pathEl->PathId); Y_VERIFY(it, "SubDomain not found"); - auto subDomainInfo = *it; - - NKikimrSubDomains::TDomainDescription * entry = Result->Record.MutablePathDescription()->MutableDomainDescription(); + auto subDomainInfo = *it; + + NKikimrSubDomains::TDomainDescription * entry = Result->Record.MutablePathDescription()->MutableDomainDescription(); - NKikimrSubDomains::TDomainKey *key = entry->MutableDomainKey(); + NKikimrSubDomains::TDomainKey *key = entry->MutableDomainKey(); entry->SetSchemeShardId_Depricated(Self->ParentDomainId.OwnerId); entry->SetPathId_Depricated(Self->ParentDomainId.LocalPathId); @@ -612,24 +612,24 @@ void TPathDescriber::DescribeDomainRoot(TPathElement::TPtr pathEl) { key->SetSchemeShard(pathEl->PathId.OwnerId); key->SetPathId(pathEl->PathId.LocalPathId); } - - entry->MutableProcessingParams()->CopyFrom(subDomainInfo->GetProcessingParams()); - - entry->SetPathsInside(subDomainInfo->GetPathsInside()); - entry->SetPathsLimit(subDomainInfo->GetSchemeLimits().MaxPaths); - entry->SetShardsInside(subDomainInfo->GetShardsInside()); - entry->SetShardsLimit(subDomainInfo->GetSchemeLimits().MaxShards); + + entry->MutableProcessingParams()->CopyFrom(subDomainInfo->GetProcessingParams()); + + entry->SetPathsInside(subDomainInfo->GetPathsInside()); + entry->SetPathsLimit(subDomainInfo->GetSchemeLimits().MaxPaths); + entry->SetShardsInside(subDomainInfo->GetShardsInside()); + entry->SetShardsLimit(subDomainInfo->GetSchemeLimits().MaxShards); entry->SetPQPartitionsInside(subDomainInfo->GetPQPartitionsInside()); entry->SetPQPartitionsLimit(subDomainInfo->GetSchemeLimits().MaxPQPartitions); - - NKikimrSubDomains::TDomainKey *resourcesKey = entry->MutableResourcesDomainKey(); - resourcesKey->SetSchemeShard(subDomainInfo->GetResourcesDomainId().OwnerId); - resourcesKey->SetPathId(subDomainInfo->GetResourcesDomainId().LocalPathId); - - NKikimrSubDomains::TDiskSpaceUsage *diskSpaceUsage = entry->MutableDiskSpaceUsage(); - diskSpaceUsage->MutableTables()->SetTotalSize(subDomainInfo->GetDiskSpaceUsage().Tables.TotalSize); - diskSpaceUsage->MutableTables()->SetDataSize(subDomainInfo->GetDiskSpaceUsage().Tables.DataSize); - diskSpaceUsage->MutableTables()->SetIndexSize(subDomainInfo->GetDiskSpaceUsage().Tables.IndexSize); + + NKikimrSubDomains::TDomainKey *resourcesKey = entry->MutableResourcesDomainKey(); + resourcesKey->SetSchemeShard(subDomainInfo->GetResourcesDomainId().OwnerId); + resourcesKey->SetPathId(subDomainInfo->GetResourcesDomainId().LocalPathId); + + NKikimrSubDomains::TDiskSpaceUsage *diskSpaceUsage = entry->MutableDiskSpaceUsage(); + diskSpaceUsage->MutableTables()->SetTotalSize(subDomainInfo->GetDiskSpaceUsage().Tables.TotalSize); + diskSpaceUsage->MutableTables()->SetDataSize(subDomainInfo->GetDiskSpaceUsage().Tables.DataSize); + diskSpaceUsage->MutableTables()->SetIndexSize(subDomainInfo->GetDiskSpaceUsage().Tables.IndexSize); if (subDomainInfo->GetDeclaredSchemeQuotas()) { entry->MutableDeclaredSchemeQuotas()->CopyFrom(*subDomainInfo->GetDeclaredSchemeQuotas()); @@ -642,8 +642,8 @@ void TPathDescriber::DescribeDomainRoot(TPathElement::TPtr pathEl) { if (subDomainInfo->GetDiskQuotaExceeded()) { entry->MutableDomainState()->SetDiskQuotaExceeded(true); } -} - +} + void TPathDescriber::DescribeDomainExtra(TPathElement::TPtr pathEl) { Y_VERIFY(pathEl->IsDomainRoot()); auto it = Self->SubDomains.FindPtr(pathEl->PathId); @@ -660,30 +660,30 @@ void TPathDescriber::DescribeDomainExtra(TPathElement::TPtr pathEl) { } } -void TPathDescriber::DescribeBlockStoreVolume(TPathId pathId, TPathElement::TPtr pathEl) { - Y_VERIFY(pathEl->IsBlockStoreVolume()); - auto it = Self->BlockStoreVolumes.FindPtr(pathId); - Y_VERIFY(it, "BlockStore volume is not found"); - TBlockStoreVolumeInfo::TPtr volume = *it; - - auto* entry = Result->Record.MutablePathDescription()->MutableBlockStoreVolumeDescription(); - entry->SetName(pathEl->Name); +void TPathDescriber::DescribeBlockStoreVolume(TPathId pathId, TPathElement::TPtr pathEl) { + Y_VERIFY(pathEl->IsBlockStoreVolume()); + auto it = Self->BlockStoreVolumes.FindPtr(pathId); + Y_VERIFY(it, "BlockStore volume is not found"); + TBlockStoreVolumeInfo::TPtr volume = *it; + + auto* entry = Result->Record.MutablePathDescription()->MutableBlockStoreVolumeDescription(); + entry->SetName(pathEl->Name); entry->SetPathId(pathId.LocalPathId); - entry->MutableVolumeConfig()->CopyFrom(volume->VolumeConfig); - entry->MutableVolumeConfig()->SetVersion(volume->AlterVersion); + entry->MutableVolumeConfig()->CopyFrom(volume->VolumeConfig); + entry->MutableVolumeConfig()->SetVersion(volume->AlterVersion); entry->SetVolumeTabletId(ui64(volume->VolumeTabletId)); - entry->SetAlterVersion(volume->AlterVersion); - entry->SetMountToken(volume->MountToken); + entry->SetAlterVersion(volume->AlterVersion); + entry->SetMountToken(volume->MountToken); entry->SetTokenVersion(volume->TokenVersion); - - const auto& tablets = volume->GetTablets(Self->ShardInfos); - for (ui32 idx = 0; idx < tablets.size(); ++idx) { - auto* part = entry->AddPartitions(); - part->SetPartitionId(idx); + + const auto& tablets = volume->GetTablets(Self->ShardInfos); + for (ui32 idx = 0; idx < tablets.size(); ++idx) { + auto* part = entry->AddPartitions(); + part->SetPartitionId(idx); part->SetTabletId(ui64(tablets[idx])); - } -} - + } +} + void TPathDescriber::DescribeFileStore(TPathId pathId, TPathElement::TPtr pathEl) { Y_VERIFY(pathEl->IsFileStore()); auto it = Self->FileStoreInfos.FindPtr(pathId); @@ -703,57 +703,57 @@ void TPathDescriber::DescribeFileStore(TPathId pathId, TPathElement::TPtr pathEl config->SetVersion(fs->Version); } -void TPathDescriber::DescribeKesus(TPathId pathId, TPathElement::TPtr pathEl) { - Y_VERIFY(pathEl->IsKesus()); - auto it = Self->KesusInfos.FindPtr(pathId); - Y_VERIFY(it, "Kesus info not found"); - TKesusInfo::TPtr kesus = *it; - - auto* entry = Result->Record.MutablePathDescription()->MutableKesus(); - entry->SetName(pathEl->Name); +void TPathDescriber::DescribeKesus(TPathId pathId, TPathElement::TPtr pathEl) { + Y_VERIFY(pathEl->IsKesus()); + auto it = Self->KesusInfos.FindPtr(pathId); + Y_VERIFY(it, "Kesus info not found"); + TKesusInfo::TPtr kesus = *it; + + auto* entry = Result->Record.MutablePathDescription()->MutableKesus(); + entry->SetName(pathEl->Name); entry->SetPathId(pathId.LocalPathId); if (kesus->KesusTabletId) { entry->SetKesusTabletId(ui64(kesus->KesusTabletId)); } - entry->MutableConfig()->CopyFrom(kesus->Config); - entry->SetVersion(kesus->Version); -} - + entry->MutableConfig()->CopyFrom(kesus->Config); + entry->SetVersion(kesus->Version); +} + void TPathDescriber::DescribeSequence(TPathId pathId, TPathElement::TPtr pathEl) { Y_VERIFY(pathEl->IsSequence()); Self->DescribeSequence(pathId, pathEl->Name, *Result->Record.MutablePathDescription()->MutableSequenceDescription()); } -void TPathDescriber::DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl) { - Y_VERIFY(pathEl->IsReplication()); - Self->DescribeReplication(pathId, pathEl->Name, *Result->Record.MutablePathDescription()->MutableReplicationDescription()); -} - +void TPathDescriber::DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl) { + Y_VERIFY(pathEl->IsReplication()); + Self->DescribeReplication(pathId, pathEl->Name, *Result->Record.MutablePathDescription()->MutableReplicationDescription()); +} + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe(const TActorContext& ctx) { TPathId pathId = Params.HasPathId() ? TPathId(Params.GetSchemeshardId(), Params.GetPathId()) : InvalidPathId; - TString pathStr = Params.GetPath(); - - TPath path = Params.HasPathId() - ? TPath::Init(pathId, Self) - : TPath::Resolve(pathStr, Self); - { - TPath::TChecker checks = path.Check(); - checks + TString pathStr = Params.GetPath(); + + TPath path = Params.HasPathId() + ? TPath::Init(pathId, Self) + : TPath::Resolve(pathStr, Self); + { + TPath::TChecker checks = path.Check(); + checks .NotEmpty(NKikimrScheme::StatusPathDoesNotExist) .IsAtLocalSchemeShard() - .IsResolved(); - - if (checks) { - if (Params.HasPathId()) { - pathStr = path.PathString(); - } else { - pathId = path.Base()->PathId; - } - } - - checks + .IsResolved(); + + if (checks) { + if (Params.HasPathId()) { + pathStr = path.PathString(); + } else { + pathId = path.Base()->PathId; + } + } + + checks .NotDeleted(); - + if (checks && !path.Base()->IsTable() && !path.Base()->IsTableIndex() && !path.Base()->IsDirectory()) { // KIKIMR-13173 // PQ BSV drop their shard before PlatStep @@ -765,10 +765,10 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe checks.IsCommonSensePath(); } - if (!checks) { - TString explain; - auto status = checks.GetStatus(&explain); - + if (!checks) { + TString explain; + auto status = checks.GetStatus(&explain); + if (Params.HasPathId()) { pathStr = path.PathString(); } @@ -778,9 +778,9 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe Self->TabletID(), pathId )); - Result->Record.SetStatus(status); - Result->Record.SetReason(explain); - + Result->Record.SetStatus(status); + Result->Record.SetReason(explain); + TPath firstExisted = path.FirstExistedParent(); FillLastExistedPrefixDescr(firstExisted); @@ -788,20 +788,20 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe DescribeDomain(firstExisted.Base()); } - return std::move(Result); - } - } - + return std::move(Result); + } + } + Result = MakeHolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder>(pathStr, Self->TabletID(), pathId); - - auto descr = Result->Record.MutablePathDescription()->MutableSelf(); - FillPathDescr(descr, path); + + auto descr = Result->Record.MutablePathDescription()->MutableSelf(); + FillPathDescr(descr, path); BuildEffectiveACL(descr, path); auto base = path.Base(); DescribeDomain(base); DescribeUserAttributes(base); DescribePathVersion(path); - + if (base->IsCreateFinished()) { switch (base->PathType) { case NKikimrSchemeOp::EPathTypeDir: @@ -848,14 +848,14 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe DescribeTableIndex(path); break; case NKikimrSchemeOp::EPathTypeCdcStream: - DescribeCdcStream(path); + DescribeCdcStream(path); break; case NKikimrSchemeOp::EPathTypeSequence: DescribeSequence(path.Base()->PathId, path.Base()); break; - case NKikimrSchemeOp::EPathTypeReplication: - DescribeReplication(path.Base()->PathId, path.Base()); - break; + case NKikimrSchemeOp::EPathTypeReplication: + DescribeReplication(path.Base()->PathId, path.Base()); + break; case NKikimrSchemeOp::EPathTypeInvalid: Y_UNREACHABLE(); } @@ -863,54 +863,54 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe // here we do not full any object specific information, like table description // nevertheless, chindren list should be set even when dir or children is being created right now DescribeChildren(path); - } - + } + Result->Record.SetStatus(NKikimrScheme::StatusSuccess); - return std::move(Result); -} - + return std::move(Result); +} + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> DescribePath( TSchemeShard* self, - const TActorContext& ctx, - TPathId pathId, + const TActorContext& ctx, + TPathId pathId, const NKikimrSchemeOp::TDescribeOptions& opts -) { +) { NKikimrSchemeOp::TDescribePath params; params.SetSchemeshardId(pathId.OwnerId); // ???? params.SetPathId(pathId.LocalPathId); - params.MutableOptions()->CopyFrom(opts); - - return TPathDescriber(self, std::move(params)).Describe(ctx); -} - + params.MutableOptions()->CopyFrom(opts); + + return TPathDescriber(self, std::move(params)).Describe(ctx); +} + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> DescribePath( TSchemeShard* self, - const TActorContext& ctx, - TPathId pathId -) { + const TActorContext& ctx, + TPathId pathId +) { NKikimrSchemeOp::TDescribeOptions options; - options.SetShowPrivateTable(true); - return DescribePath(self, ctx, pathId, options); -} - + options.SetShowPrivateTable(true); + return DescribePath(self, ctx, pathId, options); +} + void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme::TTypeRegistry* typeRegistry, bool fillConfig, bool fillBoundaries, NKikimrSchemeOp::TTableDescription* entry) const { THashMap<ui32, TString> familyNames; bool familyNamesBuilt = false; - + entry->SetTableSchemaVersion(tableInfo->AlterVersion); - entry->MutableColumns()->Reserve(tableInfo->Columns.size()); - for (auto col : tableInfo->Columns) { - const auto& cinfo = col.second; - if (cinfo.IsDropped()) - continue; - - auto colDescr = entry->AddColumns(); - colDescr->SetName(cinfo.Name); - colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType)); - colDescr->SetTypeId(cinfo.PType); - colDescr->SetId(cinfo.Id); + entry->MutableColumns()->Reserve(tableInfo->Columns.size()); + for (auto col : tableInfo->Columns) { + const auto& cinfo = col.second; + if (cinfo.IsDropped()) + continue; + + auto colDescr = entry->AddColumns(); + colDescr->SetName(cinfo.Name); + colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType)); + colDescr->SetTypeId(cinfo.PType); + colDescr->SetId(cinfo.Id); colDescr->SetNotNull(cinfo.NotNull); if (cinfo.Family != 0) { @@ -938,100 +938,100 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme colDescr->SetDefaultFromSequence(cinfo.DefaultValue); break; } - } - Y_VERIFY(!tableInfo->KeyColumnIds.empty()); + } + Y_VERIFY(!tableInfo->KeyColumnIds.empty()); entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size()); entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size()); - for (ui32 keyColId : tableInfo->KeyColumnIds) { - entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name); - entry->AddKeyColumnIds(keyColId); - } - - if (fillConfig) { - entry->MutablePartitionConfig()->CopyFrom(tableInfo->PartitionConfig()); + for (ui32 keyColId : tableInfo->KeyColumnIds) { + entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name); + entry->AddKeyColumnIds(keyColId); + } + + if (fillConfig) { + entry->MutablePartitionConfig()->CopyFrom(tableInfo->PartitionConfig()); TPartitionConfigMerger::DeduplicateColumnFamiliesById(*entry->MutablePartitionConfig()); - entry->MutablePartitionConfig()->MutableStorageRooms()->Clear(); - } - - if (fillBoundaries) { - FillTableBoundaries(tableInfo, *entry->MutableSplitBoundary()); - } - - if (tableInfo->HasTTLSettings()) { - entry->MutableTTLSettings()->CopyFrom(tableInfo->TTLSettings()); - } - - entry->SetIsBackup(tableInfo->IsBackup); -} - + entry->MutablePartitionConfig()->MutableStorageRooms()->Clear(); + } + + if (fillBoundaries) { + FillTableBoundaries(tableInfo, *entry->MutableSplitBoundary()); + } + + if (tableInfo->HasTTLSettings()) { + entry->MutableTTLSettings()->CopyFrom(tableInfo->TTLSettings()); + } + + entry->SetIsBackup(tableInfo->IsBackup); +} + void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TIndexDescription& entry) -{ - auto it = Indexes.FindPtr(pathId); - Y_VERIFY(it, "TableIndex is not found"); - TTableIndexInfo::TPtr indexInfo = *it; - - DescribeTableIndex(pathId, name, indexInfo, entry); -} - +{ + auto it = Indexes.FindPtr(pathId); + Y_VERIFY(it, "TableIndex is not found"); + TTableIndexInfo::TPtr indexInfo = *it; + + DescribeTableIndex(pathId, name, indexInfo, entry); +} + void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name, TTableIndexInfo::TPtr indexInfo, NKikimrSchemeOp::TIndexDescription& entry) -{ - Y_VERIFY(indexInfo, "Empty index info"); - - entry.SetName(name); - entry.SetLocalPathId(pathId.LocalPathId); - entry.SetPathOwnerId(pathId.OwnerId); - - entry.SetType(indexInfo->Type); - entry.SetState(indexInfo->State); - entry.SetSchemaVersion(indexInfo->AlterVersion); - - for (const auto& keyName: indexInfo->IndexKeys) { - *entry.MutableKeyColumnNames()->Add() = keyName; - } - - for (const auto& dataColumns: indexInfo->IndexDataColumns) { - *entry.MutableDataColumnNames()->Add() = dataColumns; - } - - Y_VERIFY(PathsById.contains(pathId)); - auto indexPath = PathsById.at(pathId); - - Y_VERIFY(indexPath->GetChildren().size() == 1); - const auto& indexImplPathId = indexPath->GetChildren().begin()->second; - - Y_VERIFY(Tables.contains(indexImplPathId)); - auto indexImplTable = Tables.at(indexImplPathId); - - const auto& tableStats = indexImplTable->GetStats().Aggregated; - entry.SetDataSize(tableStats.DataSize + tableStats.IndexSize); -} - +{ + Y_VERIFY(indexInfo, "Empty index info"); + + entry.SetName(name); + entry.SetLocalPathId(pathId.LocalPathId); + entry.SetPathOwnerId(pathId.OwnerId); + + entry.SetType(indexInfo->Type); + entry.SetState(indexInfo->State); + entry.SetSchemaVersion(indexInfo->AlterVersion); + + for (const auto& keyName: indexInfo->IndexKeys) { + *entry.MutableKeyColumnNames()->Add() = keyName; + } + + for (const auto& dataColumns: indexInfo->IndexDataColumns) { + *entry.MutableDataColumnNames()->Add() = dataColumns; + } + + Y_VERIFY(PathsById.contains(pathId)); + auto indexPath = PathsById.at(pathId); + + Y_VERIFY(indexPath->GetChildren().size() == 1); + const auto& indexImplPathId = indexPath->GetChildren().begin()->second; + + Y_VERIFY(Tables.contains(indexImplPathId)); + auto indexImplTable = Tables.at(indexImplPathId); + + const auto& tableStats = indexImplTable->GetStats().Aggregated; + entry.SetDataSize(tableStats.DataSize + tableStats.IndexSize); +} + void TSchemeShard::DescribeCdcStream(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TCdcStreamDescription& desc) -{ - Y_VERIFY_S(CdcStreams.contains(pathId), "Cdc stream not found" - << ": pathId# " << pathId - << ", name# " << name); - DescribeCdcStream(pathId, name, CdcStreams.at(pathId), desc); -} - +{ + Y_VERIFY_S(CdcStreams.contains(pathId), "Cdc stream not found" + << ": pathId# " << pathId + << ", name# " << name); + DescribeCdcStream(pathId, name, CdcStreams.at(pathId), desc); +} + void TSchemeShard::DescribeCdcStream(const TPathId& pathId, const TString& name, TCdcStreamInfo::TPtr info, NKikimrSchemeOp::TCdcStreamDescription& desc) -{ - Y_VERIFY_S(info, "Empty cdc stream info" - << ": pathId# " << pathId - << ", name# " << name); - - desc.SetName(name); - desc.SetMode(info->Mode); - PathIdFromPathId(pathId, desc.MutablePathId()); - desc.SetState(info->State); - desc.SetSchemaVersion(info->AlterVersion); -} - +{ + Y_VERIFY_S(info, "Empty cdc stream info" + << ": pathId# " << pathId + << ", name# " << name); + + desc.SetName(name); + desc.SetMode(info->Mode); + PathIdFromPathId(pathId, desc.MutablePathId()); + desc.SetState(info->State); + desc.SetSchemaVersion(info->AlterVersion); +} + void TSchemeShard::DescribeSequence(const TPathId& pathId, const TString& name, NKikimrSchemeOp::TSequenceDescription& desc) { @@ -1052,7 +1052,7 @@ void TSchemeShard::DescribeSequence(const TPathId& pathId, const TString& name, desc = info->Description; desc.SetName(name); - PathIdFromPathId(pathId, desc.MutablePathId()); + PathIdFromPathId(pathId, desc.MutablePathId()); desc.SetVersion(info->AlterVersion); if (info->Sharding.SequenceShardsSize() > 0) { @@ -1065,58 +1065,58 @@ void TSchemeShard::DescribeSequence(const TPathId& pathId, const TString& name, } } -void TSchemeShard::DescribeReplication(const TPathId& pathId, const TString& name, - NKikimrSchemeOp::TReplicationDescription& desc) -{ - auto it = Replications.find(pathId); - Y_VERIFY_S(it != Replications.end(), "Replication not found" - << " pathId# " << pathId - << " name# " << name); - DescribeReplication(pathId, name, it->second, desc); -} - -void TSchemeShard::DescribeReplication(const TPathId& pathId, const TString& name, TReplicationInfo::TPtr info, - NKikimrSchemeOp::TReplicationDescription& desc) -{ - Y_VERIFY_S(info, "Empty sequence info" - << " pathId# " << pathId - << " name# " << name); - - desc = info->Description; - - desc.SetName(name); - PathIdFromPathId(pathId, desc.MutablePathId()); - desc.SetVersion(info->AlterVersion); - - const auto& controllers = ResolveDomainInfo(pathId)->GetReplicationControllers(); - if (!controllers.empty()) { - const auto shardIdx = *controllers.begin(); - - Y_VERIFY(ShardInfos.contains(shardIdx)); - const auto& shardInfo = ShardInfos.at(shardIdx); - - if (shardInfo.TabletID != InvalidTabletId) { - desc.SetControllerId(ui64(shardInfo.TabletID)); - } - } -} - +void TSchemeShard::DescribeReplication(const TPathId& pathId, const TString& name, + NKikimrSchemeOp::TReplicationDescription& desc) +{ + auto it = Replications.find(pathId); + Y_VERIFY_S(it != Replications.end(), "Replication not found" + << " pathId# " << pathId + << " name# " << name); + DescribeReplication(pathId, name, it->second, desc); +} + +void TSchemeShard::DescribeReplication(const TPathId& pathId, const TString& name, TReplicationInfo::TPtr info, + NKikimrSchemeOp::TReplicationDescription& desc) +{ + Y_VERIFY_S(info, "Empty sequence info" + << " pathId# " << pathId + << " name# " << name); + + desc = info->Description; + + desc.SetName(name); + PathIdFromPathId(pathId, desc.MutablePathId()); + desc.SetVersion(info->AlterVersion); + + const auto& controllers = ResolveDomainInfo(pathId)->GetReplicationControllers(); + if (!controllers.empty()) { + const auto shardIdx = *controllers.begin(); + + Y_VERIFY(ShardInfos.contains(shardIdx)); + const auto& shardInfo = ShardInfos.at(shardIdx); + + if (shardInfo.TabletID != InvalidTabletId) { + desc.SetControllerId(ui64(shardInfo.TabletID)); + } + } +} + void TSchemeShard::FillTableBoundaries(const TTableInfo::TPtr tableInfo, google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries) { - TString errStr; - // Number of split boundaries equals to number of partitions - 1 - boundaries.Reserve(tableInfo->GetPartitions().size() - 1); - for (ui32 pi = 0; pi < tableInfo->GetPartitions().size() - 1; ++pi) { - const auto& p = tableInfo->GetPartitions()[pi]; - TSerializedCellVec endKey(p.EndOfRange); - auto boundary = boundaries.Add()->MutableKeyPrefix(); - for (ui32 ki = 0; ki < endKey.GetCells().size(); ++ki){ - const auto& c = endKey.GetCells()[ki]; - ui32 typeId = tableInfo->Columns[tableInfo->KeyColumnIds[ki]].PType; - bool ok = NMiniKQL::CellToValue(typeId, c, *boundary->AddTuple(), errStr); - Y_VERIFY(ok, "Failed to build key tuple at postition %" PRIu32 " error: %s", ki, errStr.data()); - } - } -} - + TString errStr; + // Number of split boundaries equals to number of partitions - 1 + boundaries.Reserve(tableInfo->GetPartitions().size() - 1); + for (ui32 pi = 0; pi < tableInfo->GetPartitions().size() - 1; ++pi) { + const auto& p = tableInfo->GetPartitions()[pi]; + TSerializedCellVec endKey(p.EndOfRange); + auto boundary = boundaries.Add()->MutableKeyPrefix(); + for (ui32 ki = 0; ki < endKey.GetCells().size(); ++ki){ + const auto& c = endKey.GetCells()[ki]; + ui32 typeId = tableInfo->Columns[tableInfo->KeyColumnIds[ki]].PType; + bool ok = NMiniKQL::CellToValue(typeId, c, *boundary->AddTuple(), errStr); + Y_VERIFY(ok, "Failed to build key tuple at postition %" PRIu32 " error: %s", ki, errStr.data()); + } + } +} + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.h b/ydb/core/tx/schemeshard/schemeshard_path_describer.h index 37a358827ba..7d10ed1da7a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.h @@ -1,81 +1,81 @@ -#pragma once - -#include "schemeshard_impl.h" - +#pragma once + +#include "schemeshard_impl.h" + #include <ydb/core/protos/flat_scheme_op.pb.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> - -namespace NKikimr { + +#include <util/generic/ptr.h> +#include <util/generic/string.h> + +namespace NKikimr { namespace NSchemeShard { - -class TPathDescriber { + +class TPathDescriber { void FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, - TPathElement::EPathSubType subType = TPathElement::EPathSubType::EPathSubTypeEmpty); + TPathElement::EPathSubType subType = TPathElement::EPathSubType::EPathSubTypeEmpty); void FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, const TPath& path); void FillChildDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl); - TPathElement::EPathSubType CalcPathSubType(const TPath& path); - + TPathElement::EPathSubType CalcPathSubType(const TPath& path); + void BuildEffectiveACL(NKikimrSchemeOp::TDirEntry* descr, const TPath& pathEl); - void FillLastExistedPrefixDescr(const TPath& path); - + void FillLastExistedPrefixDescr(const TPath& path); + void DescribeChildren(const TPath& path); void DescribeDir(const TPath& path); - void DescribeTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl); + void DescribeTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl); void DescribeOlapStore(TPathId pathId, TPathElement::TPtr pathEl); void DescribeOlapTable(TPathId pathId, TPathElement::TPtr pathEl); - void DescribePersQueueGroup(TPathId pathId, TPathElement::TPtr pathEl); - void DescribeRtmrVolume(TPathId pathId, TPathElement::TPtr pathEl); + void DescribePersQueueGroup(TPathId pathId, TPathElement::TPtr pathEl); + void DescribeRtmrVolume(TPathId pathId, TPathElement::TPtr pathEl); void DescribeTableIndex(const TPath& path); - void DescribeCdcStream(const TPath& path); - void DescribeSolomonVolume(TPathId pathId, TPathElement::TPtr pathEl, bool returnChannelsBinding); - void DescribeUserAttributes(TPathElement::TPtr pathEl); + void DescribeCdcStream(const TPath& path); + void DescribeSolomonVolume(TPathId pathId, TPathElement::TPtr pathEl, bool returnChannelsBinding); + void DescribeUserAttributes(TPathElement::TPtr pathEl); void DescribePathVersion(const TPath& path); void DescribeDomain(TPathElement::TPtr pathEl); void DescribeDomainRoot(TPathElement::TPtr pathEl); void DescribeDomainExtra(TPathElement::TPtr pathEl); void DescribeRevertedMigrations(TPathElement::TPtr pathEl); - void DescribeBlockStoreVolume(TPathId pathId, TPathElement::TPtr pathEl); + void DescribeBlockStoreVolume(TPathId pathId, TPathElement::TPtr pathEl); void DescribeFileStore(TPathId pathId, TPathElement::TPtr pathEl); - void DescribeKesus(TPathId pathId, TPathElement::TPtr pathEl); + void DescribeKesus(TPathId pathId, TPathElement::TPtr pathEl); void DescribeSequence(TPathId pathId, TPathElement::TPtr pathEl); - void DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl); - -public: + void DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl); + +public: explicit TPathDescriber(TSchemeShard* self, NKikimrSchemeOp::TDescribePath&& params) - : Self(self) - , Params(std::move(params)) - { - } - + : Self(self) + , Params(std::move(params)) + { + } + const NKikimrSchemeOp::TDescribePath& GetParams() const { - return Params; - } - + return Params; + } + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> Describe(const TActorContext& ctx); - -private: + +private: TSchemeShard* Self; NKikimrSchemeOp::TDescribePath Params; - + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> Result; - -}; // TPathDescriber - + +}; // TPathDescriber + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> DescribePath( TSchemeShard* self, - const TActorContext& ctx, - TPathId pathId, + const TActorContext& ctx, + TPathId pathId, const NKikimrSchemeOp::TDescribeOptions& opts -); - +); + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> DescribePath( TSchemeShard* self, - const TActorContext& ctx, - TPathId pathId -); - + const TActorContext& ctx, + TPathId pathId +); + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h index 65812e7fe7b..9912880c057 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h @@ -157,21 +157,21 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { return Attrs.size(); } - ui64 Bytes() const { - ui64 bytes = 0; - - for (const auto& [key, value] : Attrs) { - bytes += key.size(); - bytes += value.size(); - } - - return bytes; - } - + ui64 Bytes() const { + ui64 bytes = 0; + + for (const auto& [key, value] : Attrs) { + bytes += key.size(); + bytes += value.size(); + } + + return bytes; + } + bool CheckLimits(TString& errStr) const { - const ui64 bytes = Bytes(); + const ui64 bytes = Bytes(); if (bytes > TUserAttributesLimits::MaxBytes) { - errStr = Sprintf("UserArttibutes::CheckLimits: user attributes too big: %" PRIu64, bytes); + errStr = Sprintf("UserArttibutes::CheckLimits: user attributes too big: %" PRIu64, bytes); return false; } @@ -315,7 +315,7 @@ struct TPathElement : TSimpleRefCount<TPathElement> { TTxId LastTxId = InvalidTxId; ui64 DirAlterVersion = 0; - ui64 ACLVersion = 0; + ui64 ACLVersion = 0; TUserAttributes::TPtr UserAttrs; @@ -400,10 +400,10 @@ public: return PathType == EPathType::EPathTypeTableIndex; } - bool IsCdcStream() const { - return PathType == EPathType::EPathTypeCdcStream; - } - + bool IsCdcStream() const { + return PathType == EPathType::EPathTypeCdcStream; + } + bool IsTable() const { return PathType == EPathType::EPathTypeTable; } @@ -456,10 +456,10 @@ public: return PathType == EPathType::EPathTypeSequence; } - bool IsReplication() const { - return PathType == EPathType::EPathTypeReplication; - } - + bool IsReplication() const { + return PathType == EPathType::EPathTypeReplication; + } + bool IsContainer() const { return PathType == EPathType::EPathTypeDir || PathType == EPathType::EPathTypeSubDomain || PathType == EPathType::EPathTypeColumnStore; diff --git a/ydb/core/tx/schemeshard/schemeshard_private.h b/ydb/core/tx/schemeshard/schemeshard_private.h index 425bf39290f..207fd8d2c77 100644 --- a/ydb/core/tx/schemeshard/schemeshard_private.h +++ b/ydb/core/tx/schemeshard/schemeshard_private.h @@ -1,20 +1,20 @@ -#pragma once -#include "defs.h" - +#pragma once +#include "defs.h" + #include "schemeshard_identificators.h" -namespace NKikimr { +namespace NKikimr { namespace NSchemeShard { - -struct TEvPrivate { - enum EEv { - EvProgressOperation = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), - EvOperationPlanStep, + +struct TEvPrivate { + enum EEv { + EvProgressOperation = EventSpaceBegin(TKikimrEvents::ES_PRIVATE), + EvOperationPlanStep, EvCommitTenantUpdate, EvUndoTenantUpdate, - EvRunConditionalErase, + EvRunConditionalErase, EvIndexBuildBilling, - EvImportSchemeReady, + EvImportSchemeReady, EvServerlessStorageBilling, EvCleanDroppedPaths, EvCleanDroppedSubDomains, @@ -23,33 +23,33 @@ struct TEvPrivate { EvRunBackgroundCompaction, EvCompletePublication, EvCompleteBarrier, - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); - - // This event is sent by a schemeshard to itself to signal that some tx state has changed - // and it should run all the actions associated with this state - struct TEvProgressOperation: public TEventLocal<TEvProgressOperation, EvProgressOperation> { - const ui64 TxId; + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)"); + + // This event is sent by a schemeshard to itself to signal that some tx state has changed + // and it should run all the actions associated with this state + struct TEvProgressOperation: public TEventLocal<TEvProgressOperation, EvProgressOperation> { + const ui64 TxId; const ui32 TxPartId; - - TEvProgressOperation(ui64 txId, ui32 part) - : TxId(txId) - , TxPartId(part) - {} - }; - - struct TEvOperationPlan: public TEventLocal<TEvOperationPlan, EvOperationPlanStep> { - const ui64 StepId; - const ui64 TxId; - - TEvOperationPlan(ui64 step, ui64 txId) - : StepId(step) - , TxId(txId) - {} - }; - + + TEvProgressOperation(ui64 txId, ui32 part) + : TxId(txId) + , TxPartId(part) + {} + }; + + struct TEvOperationPlan: public TEventLocal<TEvOperationPlan, EvOperationPlanStep> { + const ui64 StepId; + const ui64 TxId; + + TEvOperationPlan(ui64 step, ui64 txId) + : StepId(step) + , TxId(txId) + {} + }; + struct TEvCommitTenantUpdate: public TEventLocal<TEvCommitTenantUpdate, EvCommitTenantUpdate> { TEvCommitTenantUpdate() {} @@ -60,9 +60,9 @@ struct TEvPrivate { {} }; - struct TEvRunConditionalErase: public TEventLocal<TEvRunConditionalErase, EvRunConditionalErase> { - }; - + struct TEvRunConditionalErase: public TEventLocal<TEvRunConditionalErase, EvRunConditionalErase> { + }; + struct TEvIndexBuildingMakeABill: public TEventLocal<TEvIndexBuildingMakeABill, EvIndexBuildBilling> { const ui64 BuildId; const TInstant SendAt; @@ -73,20 +73,20 @@ struct TEvPrivate { {} }; - struct TEvImportSchemeReady: public TEventLocal<TEvImportSchemeReady, EvImportSchemeReady> { - const ui64 ImportId; - const ui32 ItemIdx; - const bool Success; - const TString Error; - - TEvImportSchemeReady(ui64 id, ui32 itemIdx, bool success, const TString& error) - : ImportId(id) - , ItemIdx(itemIdx) - , Success(success) - , Error(error) - {} - }; - + struct TEvImportSchemeReady: public TEventLocal<TEvImportSchemeReady, EvImportSchemeReady> { + const ui64 ImportId; + const ui32 ItemIdx; + const bool Success; + const TString Error; + + TEvImportSchemeReady(ui64 id, ui32 itemIdx, bool success, const TString& error) + : ImportId(id) + , ItemIdx(itemIdx) + , Success(success) + , Error(error) + {} + }; + struct TEvServerlessStorageBilling: public TEventLocal<TEvServerlessStorageBilling, EvServerlessStorageBilling> { TEvServerlessStorageBilling() {} @@ -156,7 +156,7 @@ struct TEvPrivate { }; -}; // TEvPrivate - +}; // TEvPrivate + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 7dcca5add7e..f293136fe85 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -28,7 +28,7 @@ struct Schema : NIceDb::Schema { struct LastTxId : Column<12, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct DirAlterVersion : Column<13, NScheme::NTypeIds::Uint64> {}; struct UserAttrsAlterVersion : Column<14, NScheme::NTypeIds::Uint64> {}; - struct ACLVersion : Column<15, NScheme::NTypeIds::Uint64> {}; + struct ACLVersion : Column<15, NScheme::NTypeIds::Uint64> {}; struct ParentOwnerId : Column<16, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; static constexpr Type Default = InvalidOwnerId;}; using TKey = TableKey<Id>; @@ -116,22 +116,22 @@ struct Schema : NIceDb::Schema { struct AlterVersion : Column<4, NScheme::NTypeIds::Uint64> {}; struct AlterTable : Column<5, NScheme::NTypeIds::Utf8> {}; // TTableDescription struct AlterTableFull : Column<6, NScheme::NTypeIds::String> {}; // TTableDescription - struct PartitioningVersion : Column<7, NScheme::NTypeIds::Uint64> {}; - struct TTLSettings : Column<8, NScheme::NTypeIds::String> {}; - struct IsBackup : Column<9, NScheme::NTypeIds::Bool> {}; + struct PartitioningVersion : Column<7, NScheme::NTypeIds::Uint64> {}; + struct TTLSettings : Column<8, NScheme::NTypeIds::String> {}; + struct IsBackup : Column<9, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<TabId>; - using TColumns = TableColumns< - TabId, - NextColId, - PartitionConfig, - AlterVersion, - AlterTable, - AlterTableFull, - PartitioningVersion, - TTLSettings, - IsBackup - >; + using TColumns = TableColumns< + TabId, + NextColId, + PartitionConfig, + AlterVersion, + AlterTable, + AlterTableFull, + PartitioningVersion, + TTLSettings, + IsBackup + >; }; struct MigratedTables : Table<54> { @@ -144,22 +144,22 @@ struct Schema : NIceDb::Schema { struct AlterTable : Column<6, NScheme::NTypeIds::Utf8> {}; // TTableDescription struct AlterTableFull : Column<7, NScheme::NTypeIds::String> {}; // TTableDescription struct PartitioningVersion : Column<8, NScheme::NTypeIds::Uint64> {}; - struct TTLSettings : Column<9, NScheme::NTypeIds::String> {}; - struct IsBackup : Column<10, NScheme::NTypeIds::Bool> {}; + struct TTLSettings : Column<9, NScheme::NTypeIds::String> {}; + struct IsBackup : Column<10, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns< - OwnerPathId, - LocalPathId, - NextColId, - PartitionConfig, - AlterVersion, - AlterTable, - AlterTableFull, - PartitioningVersion, - TTLSettings, - IsBackup - >; + using TColumns = TableColumns< + OwnerPathId, + LocalPathId, + NextColId, + PartitionConfig, + AlterVersion, + AlterTable, + AlterTableFull, + PartitioningVersion, + TTLSettings, + IsBackup + >; }; struct Columns : Table<6> { @@ -297,89 +297,89 @@ struct Schema : NIceDb::Schema { struct Id : Column<2, NScheme::NTypeIds::Uint64> {}; struct RangeEnd : Column<3, NScheme::NTypeIds::String> { using Type = TString; }; struct DatashardIdx : Column<4, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx; }; - struct LastCondErase : Column<5, NScheme::NTypeIds::Uint64> {}; - struct NextCondErase : Column<6, NScheme::NTypeIds::Uint64> {}; + struct LastCondErase : Column<5, NScheme::NTypeIds::Uint64> {}; + struct NextCondErase : Column<6, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<TabId, Id>; - using TColumns = TableColumns<TabId, Id, RangeEnd, DatashardIdx, LastCondErase, NextCondErase>; - }; - - struct TablePartitionStats: Table<101> { // HACK: Force to use policy with multiple levels - // key - struct TableOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; - struct TableLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; - struct PartitionId: Column<3, NScheme::NTypeIds::Uint64> {}; - - // seqno - struct SeqNoGeneration: Column<4, NScheme::NTypeIds::Uint64> {}; - struct SeqNoRound: Column<5, NScheme::NTypeIds::Uint64> {}; - - // stats - struct RowCount: Column<6, NScheme::NTypeIds::Uint64> {}; - struct DataSize: Column<7, NScheme::NTypeIds::Uint64> {}; - struct IndexSize: Column<8, NScheme::NTypeIds::Uint64> {}; - - struct LastAccessTime: Column<9, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; - struct LastUpdateTime: Column<10, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; - - struct ImmediateTxCompleted: Column<11, NScheme::NTypeIds::Uint64> {}; - struct PlannedTxCompleted: Column<12, NScheme::NTypeIds::Uint64> {}; - struct TxRejectedByOverload: Column<13, NScheme::NTypeIds::Uint64> {}; - struct TxRejectedBySpace: Column<14, NScheme::NTypeIds::Uint64> {}; - struct TxCompleteLag: Column<15, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; - struct InFlightTxCount: Column<16, NScheme::NTypeIds::Uint64> {}; - - struct RowUpdates: Column<17, NScheme::NTypeIds::Uint64> {}; - struct RowDeletes: Column<18, NScheme::NTypeIds::Uint64> {}; - struct RowReads: Column<19, NScheme::NTypeIds::Uint64> {}; - struct RangeReads: Column<20, NScheme::NTypeIds::Uint64> {}; - struct RangeReadRows: Column<21, NScheme::NTypeIds::Uint64> {}; - - struct CPU: Column<22, NScheme::NTypeIds::Uint64> {}; - struct Memory: Column<23, NScheme::NTypeIds::Uint64> {}; - struct Network: Column<24, NScheme::NTypeIds::Uint64> {}; - struct Storage: Column<25, NScheme::NTypeIds::Uint64> {}; - struct ReadThroughput: Column<26, NScheme::NTypeIds::Uint64> {}; - struct WriteThroughput: Column<27, NScheme::NTypeIds::Uint64> {}; - struct ReadIops: Column<28, NScheme::NTypeIds::Uint64> {}; - struct WriteIops: Column<29, NScheme::NTypeIds::Uint64> {}; - - // PartCount, PartOwners & ShardState are volatile data - - using TKey = TableKey<TableOwnerId, TableLocalId, PartitionId>; - using TColumns = TableColumns< - TableOwnerId, - TableLocalId, - PartitionId, - SeqNoGeneration, - SeqNoRound, - RowCount, - DataSize, - IndexSize, - LastAccessTime, - LastUpdateTime, - ImmediateTxCompleted, - PlannedTxCompleted, - TxRejectedByOverload, - TxRejectedBySpace, - TxCompleteLag, - InFlightTxCount, - RowUpdates, - RowDeletes, - RowReads, - RangeReads, - RangeReadRows, - CPU, - Memory, - Network, - Storage, - ReadThroughput, - WriteThroughput, - ReadIops, - WriteIops - >; - }; - + using TColumns = TableColumns<TabId, Id, RangeEnd, DatashardIdx, LastCondErase, NextCondErase>; + }; + + struct TablePartitionStats: Table<101> { // HACK: Force to use policy with multiple levels + // key + struct TableOwnerId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; + struct TableLocalId: Column<2, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; + struct PartitionId: Column<3, NScheme::NTypeIds::Uint64> {}; + + // seqno + struct SeqNoGeneration: Column<4, NScheme::NTypeIds::Uint64> {}; + struct SeqNoRound: Column<5, NScheme::NTypeIds::Uint64> {}; + + // stats + struct RowCount: Column<6, NScheme::NTypeIds::Uint64> {}; + struct DataSize: Column<7, NScheme::NTypeIds::Uint64> {}; + struct IndexSize: Column<8, NScheme::NTypeIds::Uint64> {}; + + struct LastAccessTime: Column<9, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; + struct LastUpdateTime: Column<10, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; + + struct ImmediateTxCompleted: Column<11, NScheme::NTypeIds::Uint64> {}; + struct PlannedTxCompleted: Column<12, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOverload: Column<13, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedBySpace: Column<14, NScheme::NTypeIds::Uint64> {}; + struct TxCompleteLag: Column<15, NScheme::NTypeIds::Uint64> { using Type = TInstant::TValue; }; + struct InFlightTxCount: Column<16, NScheme::NTypeIds::Uint64> {}; + + struct RowUpdates: Column<17, NScheme::NTypeIds::Uint64> {}; + struct RowDeletes: Column<18, NScheme::NTypeIds::Uint64> {}; + struct RowReads: Column<19, NScheme::NTypeIds::Uint64> {}; + struct RangeReads: Column<20, NScheme::NTypeIds::Uint64> {}; + struct RangeReadRows: Column<21, NScheme::NTypeIds::Uint64> {}; + + struct CPU: Column<22, NScheme::NTypeIds::Uint64> {}; + struct Memory: Column<23, NScheme::NTypeIds::Uint64> {}; + struct Network: Column<24, NScheme::NTypeIds::Uint64> {}; + struct Storage: Column<25, NScheme::NTypeIds::Uint64> {}; + struct ReadThroughput: Column<26, NScheme::NTypeIds::Uint64> {}; + struct WriteThroughput: Column<27, NScheme::NTypeIds::Uint64> {}; + struct ReadIops: Column<28, NScheme::NTypeIds::Uint64> {}; + struct WriteIops: Column<29, NScheme::NTypeIds::Uint64> {}; + + // PartCount, PartOwners & ShardState are volatile data + + using TKey = TableKey<TableOwnerId, TableLocalId, PartitionId>; + using TColumns = TableColumns< + TableOwnerId, + TableLocalId, + PartitionId, + SeqNoGeneration, + SeqNoRound, + RowCount, + DataSize, + IndexSize, + LastAccessTime, + LastUpdateTime, + ImmediateTxCompleted, + PlannedTxCompleted, + TxRejectedByOverload, + TxRejectedBySpace, + TxCompleteLag, + InFlightTxCount, + RowUpdates, + RowDeletes, + RowReads, + RangeReads, + RangeReadRows, + CPU, + Memory, + Network, + Storage, + ReadThroughput, + WriteThroughput, + ReadIops, + WriteIops + >; + }; + struct MigratedTablePartitions : Table<56> { struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId;}; @@ -388,11 +388,11 @@ struct Schema : NIceDb::Schema { struct RangeEnd : Column<4, NScheme::NTypeIds::String> { using Type = TString; }; struct OwnerShardIdx : Column<5, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; struct LocalShardIdx : Column<6, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx; }; - struct LastCondErase : Column<7, NScheme::NTypeIds::Uint64> {}; - struct NextCondErase : Column<8, NScheme::NTypeIds::Uint64> {}; + struct LastCondErase : Column<7, NScheme::NTypeIds::Uint64> {}; + struct NextCondErase : Column<8, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<OwnerPathId, LocalPathId, Id>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, Id, RangeEnd, OwnerShardIdx, LocalShardIdx, LastCondErase, NextCondErase>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, Id, RangeEnd, OwnerShardIdx, LocalShardIdx, LastCondErase, NextCondErase>; }; struct PersQueueGroups : Table<9> { @@ -404,14 +404,14 @@ struct Schema : NIceDb::Schema { struct TotalGroupCount : Column<6, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey<PathId>; - using TColumns = TableColumns< - PathId, - TabletConfig, - MaxPQPerShard, - AlterVersion, - NextPartitionId, - TotalGroupCount - >; + using TColumns = TableColumns< + PathId, + TabletConfig, + MaxPQPerShard, + AlterVersion, + NextPartitionId, + TotalGroupCount + >; }; struct PersQueueGroupAlters : Table<12> { @@ -421,18 +421,18 @@ struct Schema : NIceDb::Schema { struct AlterVersion : Column<4, NScheme::NTypeIds::Uint64> {}; struct NextPartitionId : Column<6, NScheme::NTypeIds::Uint32> {}; struct TotalGroupCount : Column<5, NScheme::NTypeIds::Uint32> {}; - struct BootstrapConfig : Column<7, NScheme::NTypeIds::String> { using Type = TString; }; + struct BootstrapConfig : Column<7, NScheme::NTypeIds::String> { using Type = TString; }; using TKey = TableKey<PathId>; - using TColumns = TableColumns< - PathId, - TabletConfig, - MaxPQPerShard, - AlterVersion, - NextPartitionId, - TotalGroupCount, - BootstrapConfig - >; + using TColumns = TableColumns< + PathId, + TabletConfig, + MaxPQPerShard, + AlterVersion, + NextPartitionId, + TotalGroupCount, + BootstrapConfig + >; }; struct PersQueues : Table<10> { @@ -441,11 +441,11 @@ struct Schema : NIceDb::Schema { struct ShardIdx : Column<3, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx; }; struct AlterVersion : Column<4, NScheme::NTypeIds::Uint64> {}; struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; - struct RangeBegin : Column<6, NScheme::NTypeIds::String> { using Type = TString; }; - struct RangeEnd : Column<7, NScheme::NTypeIds::String> { using Type = TString; }; + struct RangeBegin : Column<6, NScheme::NTypeIds::String> { using Type = TString; }; + struct RangeEnd : Column<7, NScheme::NTypeIds::String> { using Type = TString; }; using TKey = TableKey<PathId, PqId>; - using TColumns = TableColumns<PathId, PqId, ShardIdx, AlterVersion, GroupId, RangeBegin, RangeEnd>; + using TColumns = TableColumns<PathId, PqId, ShardIdx, AlterVersion, GroupId, RangeBegin, RangeEnd>; }; struct RtmrVolumes : Table<20> { @@ -525,12 +525,12 @@ struct Schema : NIceDb::Schema { struct PathId: Column<1, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; struct TableName: Column<2, NScheme::NTypeIds::Utf8> {}; struct YTSettings: Column<3, NScheme::NTypeIds::String> {}; - struct S3Settings: Column<6, NScheme::NTypeIds::String> {}; - struct TableDescription : Column<7, NScheme::NTypeIds::String> {}; - struct NumberOfRetries : Column<8, NScheme::NTypeIds::Uint32> {}; - struct ScanSettings : Column<9, NScheme::NTypeIds::String> {}; - struct NeedToBill: Column<10, NScheme::NTypeIds::Bool> {}; - // deprecated + struct S3Settings: Column<6, NScheme::NTypeIds::String> {}; + struct TableDescription : Column<7, NScheme::NTypeIds::String> {}; + struct NumberOfRetries : Column<8, NScheme::NTypeIds::Uint32> {}; + struct ScanSettings : Column<9, NScheme::NTypeIds::String> {}; + struct NeedToBill: Column<10, NScheme::NTypeIds::Bool> {}; + // deprecated struct CreateDestinationFlag: Column<4, NScheme::NTypeIds::Bool> {}; struct EraseOldDataFlag: Column<5, NScheme::NTypeIds::Bool> {}; @@ -540,13 +540,13 @@ struct Schema : NIceDb::Schema { TableName, YTSettings, CreateDestinationFlag, - EraseOldDataFlag, - S3Settings, - TableDescription, - NumberOfRetries, - ScanSettings, - NeedToBill - >; + EraseOldDataFlag, + S3Settings, + TableDescription, + NumberOfRetries, + ScanSettings, + NeedToBill + >; }; struct MigratedBackupSettings: Table<64> { @@ -555,12 +555,12 @@ struct Schema : NIceDb::Schema { struct TableName: Column<3, NScheme::NTypeIds::Utf8> {}; struct YTSettings: Column<4, NScheme::NTypeIds::String> {}; - struct S3Settings: Column<7, NScheme::NTypeIds::String> {}; - struct TableDescription : Column<8, NScheme::NTypeIds::String> {}; - struct NumberOfRetries : Column<9, NScheme::NTypeIds::Uint32> {}; - struct ScanSettings : Column<10, NScheme::NTypeIds::String> {}; - struct NeedToBill: Column<11, NScheme::NTypeIds::Bool> {}; - // deprecated + struct S3Settings: Column<7, NScheme::NTypeIds::String> {}; + struct TableDescription : Column<8, NScheme::NTypeIds::String> {}; + struct NumberOfRetries : Column<9, NScheme::NTypeIds::Uint32> {}; + struct ScanSettings : Column<10, NScheme::NTypeIds::String> {}; + struct NeedToBill: Column<11, NScheme::NTypeIds::Bool> {}; + // deprecated struct CreateDestinationFlag: Column<5, NScheme::NTypeIds::Bool> {}; struct EraseOldDataFlag: Column<6, NScheme::NTypeIds::Bool> {}; @@ -571,13 +571,13 @@ struct Schema : NIceDb::Schema { TableName, YTSettings, CreateDestinationFlag, - EraseOldDataFlag, - S3Settings, - TableDescription, - NumberOfRetries, - ScanSettings, - NeedToBill - >; + EraseOldDataFlag, + S3Settings, + TableDescription, + NumberOfRetries, + ScanSettings, + NeedToBill + >; }; struct CompletedBackups: Table<16> { @@ -589,9 +589,9 @@ struct Schema : NIceDb::Schema { struct TotalShardCount: Column<5, NScheme::NTypeIds::Uint32> {}; struct StartTime: Column<6, NScheme::NTypeIds::Uint64> {}; struct YTSettings: Column<7, NScheme::NTypeIds::String> {}; - struct S3Settings: Column<9, NScheme::NTypeIds::String> {}; + struct S3Settings: Column<9, NScheme::NTypeIds::String> {}; struct DataTotalSize: Column<8, NScheme::NTypeIds::Uint64> {}; - struct Kind: Column<10, NScheme::NTypeIds::Byte> {}; + struct Kind: Column<10, NScheme::NTypeIds::Byte> {}; using TKey = TableKey<PathId, TxId, DateTimeOfCompletion>; using TColumns = TableColumns< @@ -602,9 +602,9 @@ struct Schema : NIceDb::Schema { TotalShardCount, StartTime, YTSettings, - DataTotalSize, - S3Settings, - Kind + DataTotalSize, + S3Settings, + Kind >; }; @@ -618,9 +618,9 @@ struct Schema : NIceDb::Schema { struct TotalShardCount: Column<7, NScheme::NTypeIds::Uint32> {}; struct StartTime: Column<8, NScheme::NTypeIds::Uint64> {}; struct YTSettings: Column<9, NScheme::NTypeIds::String> {}; - struct S3Settings: Column<11, NScheme::NTypeIds::String> {}; + struct S3Settings: Column<11, NScheme::NTypeIds::String> {}; struct DataTotalSize: Column<10, NScheme::NTypeIds::Uint64> {}; - struct Kind: Column<12, NScheme::NTypeIds::Byte> {}; + struct Kind: Column<12, NScheme::NTypeIds::Byte> {}; using TKey = TableKey<OwnerPathId, LocalPathId, TxId, DateTimeOfCompletion>; @@ -633,13 +633,13 @@ struct Schema : NIceDb::Schema { TotalShardCount, StartTime, YTSettings, - DataTotalSize, - S3Settings, - Kind - >; + DataTotalSize, + S3Settings, + Kind + >; }; - // *ShardBackupStatus is deprecated, use TxShardStatus instead + // *ShardBackupStatus is deprecated, use TxShardStatus instead struct ShardBackupStatus: Table<17> { struct TxId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct ShardIdx : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx; }; @@ -670,28 +670,28 @@ struct Schema : NIceDb::Schema { >; }; - struct TxShardStatus : Table<87> { - struct TxId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; - struct OwnerShardId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; - struct LocalShardId : Column<3, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx;}; - - struct Success : Column<4, NScheme::NTypeIds::Bool> {}; - struct Error : Column<5, NScheme::NTypeIds::Utf8> {}; - struct BytesProcessed : Column<6, NScheme::NTypeIds::Uint64> {}; - struct RowsProcessed : Column<7, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<TxId, OwnerShardId, LocalShardId>; - using TColumns = TableColumns< - TxId, - OwnerShardId, - LocalShardId, - Success, - Error, - BytesProcessed, - RowsProcessed - >; - }; - + struct TxShardStatus : Table<87> { + struct TxId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; + struct OwnerShardId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; + struct LocalShardId : Column<3, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx;}; + + struct Success : Column<4, NScheme::NTypeIds::Bool> {}; + struct Error : Column<5, NScheme::NTypeIds::Utf8> {}; + struct BytesProcessed : Column<6, NScheme::NTypeIds::Uint64> {}; + struct RowsProcessed : Column<7, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<TxId, OwnerShardId, LocalShardId>; + using TColumns = TableColumns< + TxId, + OwnerShardId, + LocalShardId, + Success, + Error, + BytesProcessed, + RowsProcessed + >; + }; + struct SubDomains: Table<18> { struct PathId : Column<1, NScheme::NTypeIds::Uint64> {}; struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; @@ -702,17 +702,17 @@ struct Schema : NIceDb::Schema { struct ChildrenLimit : Column<7, NScheme::NTypeIds::Uint64> {}; struct ShardsLimit : Column<8, NScheme::NTypeIds::Uint64> {}; struct PathShardsLimit : Column<9, NScheme::NTypeIds::Uint64> {}; - struct TableColumnsLimit : Column<10, NScheme::NTypeIds::Uint64> {}; - struct TableColumnNameLengthLimit : Column<11, NScheme::NTypeIds::Uint64> {}; - struct TableKeyColumnsLimit : Column<12, NScheme::NTypeIds::Uint64> {}; - struct TableIndicesLimit : Column<13, NScheme::NTypeIds::Uint64> {}; - struct AclByteSizeLimit : Column<14, NScheme::NTypeIds::Uint64> {}; - struct ConsistentCopyingTargetsLimit: Column<15, NScheme::NTypeIds::Uint64> {}; + struct TableColumnsLimit : Column<10, NScheme::NTypeIds::Uint64> {}; + struct TableColumnNameLengthLimit : Column<11, NScheme::NTypeIds::Uint64> {}; + struct TableKeyColumnsLimit : Column<12, NScheme::NTypeIds::Uint64> {}; + struct TableIndicesLimit : Column<13, NScheme::NTypeIds::Uint64> {}; + struct AclByteSizeLimit : Column<14, NScheme::NTypeIds::Uint64> {}; + struct ConsistentCopyingTargetsLimit: Column<15, NScheme::NTypeIds::Uint64> {}; struct PathElementLength: Column<16, NScheme::NTypeIds::Uint64> {}; struct ExtraPathSymbolsAllowed: Column<17, NScheme::NTypeIds::Utf8> {}; - struct ResourcesDomainOwnerPathId : Column<18, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; static constexpr Type Default = InvalidOwnerId; }; - struct ResourcesDomainLocalPathId : Column<19, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; static constexpr Type Default = InvalidLocalPathId; }; - struct SharedHiveId : Column<20, NScheme::NTypeIds::Uint64> { using Type = TTabletId; static constexpr Type Default = InvalidTabletId; }; + struct ResourcesDomainOwnerPathId : Column<18, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; static constexpr Type Default = InvalidOwnerId; }; + struct ResourcesDomainLocalPathId : Column<19, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; static constexpr Type Default = InvalidLocalPathId; }; + struct SharedHiveId : Column<20, NScheme::NTypeIds::Uint64> { using Type = TTabletId; static constexpr Type Default = InvalidTabletId; }; struct DeclaredSchemeQuotas : Column<21, NScheme::NTypeIds::String> {}; struct PQPartitionsLimit : Column<22, NScheme::NTypeIds::Uint64> {}; struct DatabaseQuotas : Column<23, NScheme::NTypeIds::String> {}; @@ -730,17 +730,17 @@ struct Schema : NIceDb::Schema { PathsLimit, ChildrenLimit, ShardsLimit, - PathShardsLimit, - TableColumnsLimit, - TableColumnNameLengthLimit, - TableKeyColumnsLimit, - TableIndicesLimit, - AclByteSizeLimit, + PathShardsLimit, + TableColumnsLimit, + TableColumnNameLengthLimit, + TableKeyColumnsLimit, + TableIndicesLimit, + AclByteSizeLimit, ConsistentCopyingTargetsLimit, PathElementLength, - ExtraPathSymbolsAllowed, - ResourcesDomainOwnerPathId, - ResourcesDomainLocalPathId, + ExtraPathSymbolsAllowed, + ResourcesDomainOwnerPathId, + ResourcesDomainLocalPathId, SharedHiveId, DeclaredSchemeQuotas, PQPartitionsLimit, @@ -764,9 +764,9 @@ struct Schema : NIceDb::Schema { struct AlterVersion: Column<2, NScheme::NTypeIds::Uint64> {}; struct PlanResolution: Column<3, NScheme::NTypeIds::Uint64> {}; struct TimeCastBuckets: Column<4, NScheme::NTypeIds::Uint32> {}; - struct ResourcesDomainOwnerPathId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; static constexpr Type Default = InvalidOwnerId; }; - struct ResourcesDomainLocalPathId : Column<6, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; static constexpr Type Default = InvalidLocalPathId; }; - struct SharedHiveId : Column<7, NScheme::NTypeIds::Uint64> { using Type = TTabletId; static constexpr Type Default = InvalidTabletId; }; + struct ResourcesDomainOwnerPathId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; static constexpr Type Default = InvalidOwnerId; }; + struct ResourcesDomainLocalPathId : Column<6, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; static constexpr Type Default = InvalidLocalPathId; }; + struct SharedHiveId : Column<7, NScheme::NTypeIds::Uint64> { using Type = TTabletId; static constexpr Type Default = InvalidTabletId; }; struct DeclaredSchemeQuotas : Column<8, NScheme::NTypeIds::String> {}; struct DatabaseQuotas : Column<9, NScheme::NTypeIds::String> {}; @@ -775,9 +775,9 @@ struct Schema : NIceDb::Schema { PathId, AlterVersion, PlanResolution, - TimeCastBuckets, - ResourcesDomainOwnerPathId, - ResourcesDomainLocalPathId, + TimeCastBuckets, + ResourcesDomainOwnerPathId, + ResourcesDomainLocalPathId, SharedHiveId, DeclaredSchemeQuotas, DatabaseQuotas @@ -1090,65 +1090,65 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<TxId, ShardOwnerId, ShardLocalIdx, Operation, TxPartId>; }; - struct Exports : Table<45> { - struct Id: Column<1, NScheme::NTypeIds::Uint64> {}; - struct Uid: Column<2, NScheme::NTypeIds::Utf8> {}; - struct Kind: Column<12, NScheme::NTypeIds::Byte> {}; - struct Settings: Column<3, NScheme::NTypeIds::String> {}; - struct DomainPathId: Column<4, NScheme::NTypeIds::Uint64> {}; - struct UserSID : Column<13, NScheme::NTypeIds::Utf8> {}; - struct Items: Column<5, NScheme::NTypeIds::Uint32> {}; + struct Exports : Table<45> { + struct Id: Column<1, NScheme::NTypeIds::Uint64> {}; + struct Uid: Column<2, NScheme::NTypeIds::Utf8> {}; + struct Kind: Column<12, NScheme::NTypeIds::Byte> {}; + struct Settings: Column<3, NScheme::NTypeIds::String> {}; + struct DomainPathId: Column<4, NScheme::NTypeIds::Uint64> {}; + struct UserSID : Column<13, NScheme::NTypeIds::Utf8> {}; + struct Items: Column<5, NScheme::NTypeIds::Uint32> {}; struct ExportPathId: Column<6, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; - struct State: Column<7, NScheme::NTypeIds::Byte> {}; + struct State: Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId: Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; - struct Issue: Column<9, NScheme::NTypeIds::Utf8> {}; + struct Issue: Column<9, NScheme::NTypeIds::Utf8> {}; struct ExportOwnerPathId: Column<10, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; struct DomainPathOwnerId: Column<11, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; - - - using TKey = TableKey<Id>; - using TColumns = TableColumns< - Id, - Uid, - Settings, - DomainPathId, - Items, - ExportPathId, - State, - WaitTxId, + + + using TKey = TableKey<Id>; + using TColumns = TableColumns< + Id, + Uid, + Settings, + DomainPathId, + Items, + ExportPathId, + State, + WaitTxId, Issue, ExportOwnerPathId, - DomainPathOwnerId, - Kind, - UserSID - >; - }; - - struct ExportItems : Table<46> { - struct ExportId: Column<1, NScheme::NTypeIds::Uint64> {}; - struct Index: Column<2, NScheme::NTypeIds::Uint32> {}; - struct SourcePathName: Column<3, NScheme::NTypeIds::Utf8> {}; + DomainPathOwnerId, + Kind, + UserSID + >; + }; + + struct ExportItems : Table<46> { + struct ExportId: Column<1, NScheme::NTypeIds::Uint64> {}; + struct Index: Column<2, NScheme::NTypeIds::Uint32> {}; + struct SourcePathName: Column<3, NScheme::NTypeIds::Utf8> {}; struct SourcePathId: Column<4, NScheme::NTypeIds::Uint64> {using Type = TLocalPathId;}; - - struct State: Column<5, NScheme::NTypeIds::Byte> {}; + + struct State: Column<5, NScheme::NTypeIds::Byte> {}; struct BackupTxId: Column<6, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; - struct Issue: Column<7, NScheme::NTypeIds::Utf8> {}; + struct Issue: Column<7, NScheme::NTypeIds::Utf8> {}; struct SourceOwnerPathId: Column<8, NScheme::NTypeIds::Uint64> {using Type = TOwnerId;}; - - using TKey = TableKey<ExportId, Index>; - using TColumns = TableColumns< - ExportId, - Index, - SourcePathName, - SourcePathId, - State, - BackupTxId, + + using TKey = TableKey<ExportId, Index>; + using TColumns = TableColumns< + ExportId, + Index, + SourcePathName, + SourcePathId, + State, + BackupTxId, Issue, SourceOwnerPathId - >; - }; - + >; + }; + struct TableShardPartitionConfigs : Table<47> { struct ShardIdx : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalShardIdx; }; struct PartitionConfig : Column<2, NScheme::NTypeIds::String> {}; @@ -1171,15 +1171,15 @@ struct Schema : NIceDb::Schema { PartitionConfig>; }; - struct PublishingPaths : Table<48> { + struct PublishingPaths : Table<48> { struct TxId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; - struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Version : Column<3, NScheme::NTypeIds::Uint64> {}; - - using TKey = TableKey<TxId, PathId, Version>; - using TColumns = TableColumns<TxId, PathId, Version>; - }; - + struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Version : Column<3, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<TxId, PathId, Version>; + using TColumns = TableColumns<TxId, PathId, Version>; + }; + struct MigratedPublishingPaths : Table<59> { struct TxId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct PathOwnerId : Column<2, NScheme::NTypeIds::Uint64> {}; @@ -1389,75 +1389,75 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<PathOwnerId, PathLocalId, DataColumnId, DataColumnName>; }; - struct RestoreTasks : Table<82> { - struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; - struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId;}; - struct Task : Column<3, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns< - OwnerPathId, - LocalPathId, - Task - >; - }; - - struct Imports : Table<85> { - struct Id : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Uid : Column<2, NScheme::NTypeIds::Utf8> {}; - struct Kind : Column<3, NScheme::NTypeIds::Byte> {}; - struct Settings : Column<4, NScheme::NTypeIds::String> {}; - struct DomainPathOwnerId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; - struct DomainPathLocalId : Column<6, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct UserSID : Column<10, NScheme::NTypeIds::Utf8> {}; - struct Items : Column<7, NScheme::NTypeIds::Uint32> {}; - - struct State : Column<8, NScheme::NTypeIds::Byte> {}; - struct Issue : Column<9, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<Id>; - using TColumns = TableColumns< - Id, - Uid, - Kind, - Settings, - DomainPathOwnerId, - DomainPathLocalId, - Items, - State, - Issue, - UserSID - >; - }; - - struct ImportItems : Table<86> { - struct ImportId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct Index : Column<2, NScheme::NTypeIds::Uint32> {}; - struct DstPathName : Column<3, NScheme::NTypeIds::Utf8> {}; - struct DstPathOwnerId : Column<4, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; - struct DstPathLocalId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct Scheme : Column<6, NScheme::NTypeIds::String> {}; - - struct State : Column<7, NScheme::NTypeIds::Byte> {}; - struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; - struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; - struct Issue : Column<10, NScheme::NTypeIds::Utf8> {}; - - using TKey = TableKey<ImportId, Index>; - using TColumns = TableColumns< - ImportId, - Index, - DstPathName, - DstPathOwnerId, - DstPathLocalId, - Scheme, - State, - WaitTxId, - NextIndexIdx, - Issue - >; - }; - + struct RestoreTasks : Table<82> { + struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId;}; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId;}; + struct Task : Column<3, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<OwnerPathId, LocalPathId>; + using TColumns = TableColumns< + OwnerPathId, + LocalPathId, + Task + >; + }; + + struct Imports : Table<85> { + struct Id : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Uid : Column<2, NScheme::NTypeIds::Utf8> {}; + struct Kind : Column<3, NScheme::NTypeIds::Byte> {}; + struct Settings : Column<4, NScheme::NTypeIds::String> {}; + struct DomainPathOwnerId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct DomainPathLocalId : Column<6, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct UserSID : Column<10, NScheme::NTypeIds::Utf8> {}; + struct Items : Column<7, NScheme::NTypeIds::Uint32> {}; + + struct State : Column<8, NScheme::NTypeIds::Byte> {}; + struct Issue : Column<9, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<Id>; + using TColumns = TableColumns< + Id, + Uid, + Kind, + Settings, + DomainPathOwnerId, + DomainPathLocalId, + Items, + State, + Issue, + UserSID + >; + }; + + struct ImportItems : Table<86> { + struct ImportId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct Index : Column<2, NScheme::NTypeIds::Uint32> {}; + struct DstPathName : Column<3, NScheme::NTypeIds::Utf8> {}; + struct DstPathOwnerId : Column<4, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct DstPathLocalId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct Scheme : Column<6, NScheme::NTypeIds::String> {}; + + struct State : Column<7, NScheme::NTypeIds::Byte> {}; + struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; + struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; + struct Issue : Column<10, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<ImportId, Index>; + using TColumns = TableColumns< + ImportId, + Index, + DstPathName, + DstPathOwnerId, + DstPathLocalId, + Scheme, + State, + WaitTxId, + NextIndexIdx, + Issue + >; + }; + struct OlapStores : Table<88> { struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; @@ -1526,28 +1526,28 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<SidName, SidMember>; }; - struct CdcStream : Table<95> { - struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; - struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + struct CdcStream : Table<95> { + struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; struct State : Column<4, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamState; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamStateInvalid; }; struct Mode : Column<5, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamMode; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamModeInvalid; }; - - using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode>; - }; - - struct CdcStreamAlterData : Table<96> { - struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; - struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey<OwnerPathId, LocalPathId>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode>; + }; + + struct CdcStreamAlterData : Table<96> { + struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; struct State : Column<4, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamState; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamStateInvalid; }; struct Mode : Column<5, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamMode; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamModeInvalid; }; - - using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode>; - }; - + + using TKey = TableKey<OwnerPathId, LocalPathId>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode>; + }; + struct Sequences : Table<97> { struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; @@ -1568,24 +1568,24 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<PathId, AlterVersion, Description, Sharding>; }; - struct Replications : Table<99> { - struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Description : Column<3, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<PathId>; - using TColumns = TableColumns<PathId, AlterVersion, Description>; - }; - - struct ReplicationsAlterData : Table<100> { - struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; - struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Description : Column<3, NScheme::NTypeIds::String> {}; - - using TKey = TableKey<PathId>; - using TColumns = TableColumns<PathId, AlterVersion, Description>; - }; - + struct Replications : Table<99> { + struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Description : Column<3, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<PathId>; + using TColumns = TableColumns<PathId, AlterVersion, Description>; + }; + + struct ReplicationsAlterData : Table<100> { + struct PathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Description : Column<3, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<PathId>; + using TColumns = TableColumns<PathId, AlterVersion, Description>; + }; + using TTables = SchemaTables< Paths, TxInFlight, @@ -1629,10 +1629,10 @@ struct Schema : NIceDb::Schema { TableIndexKeysAlterData, TxInFlightV2, TxDependenciesV2, - TxShardsV2, - Exports, - ExportItems, - TableShardPartitionConfigs, + TxShardsV2, + Exports, + ExportItems, + TableShardPartitionConfigs, PublishingPaths, FillIndexDesc, MigratedPaths, @@ -1661,18 +1661,18 @@ struct Schema : NIceDb::Schema { LongLocks, IndexBuildShardStatus, AlterSolomonVolumes, - AlterSolomonPartitions, + AlterSolomonPartitions, TablePartitionStats, SubDomainSchemeQuotas, FileStoreInfos, FileStoreAlters, TableIndexDataColumns, - TableIndexDataColumnsAlterData, + TableIndexDataColumnsAlterData, RestoreTasks, MigratedKesusInfos, - MigratedKesusAlters, - Imports, - ImportItems, + MigratedKesusAlters, + Imports, + ImportItems, TxShardStatus, OlapStores, OlapStoresAlters, @@ -1680,13 +1680,13 @@ struct Schema : NIceDb::Schema { OlapTablesAlters, LoginKeys, LoginSids, - LoginSidMembers, - CdcStream, + LoginSidMembers, + CdcStream, CdcStreamAlterData, Sequences, - SequencesAlters, - Replications, - ReplicationsAlterData + SequencesAlters, + Replications, + ReplicationsAlterData >; static constexpr ui64 SysParam_NextPathId = 1; diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index e5ec201110e..41a8045155a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -89,27 +89,27 @@ struct TTxState { item(TxCreateFileStore, 44) \ item(TxAlterFileStore, 45) \ item(TxDropFileStore, 46) \ - item(TxRestore, 47) \ + item(TxRestore, 47) \ item(TxCreateOlapStore, 48) \ item(TxAlterOlapStore, 49) \ item(TxDropOlapStore, 50) \ item(TxCreateOlapTable, 51) \ item(TxAlterOlapTable, 52) \ item(TxDropOlapTable, 53) \ - item(TxCreateCdcStream, 54) \ - item(TxCreateCdcStreamAtTable, 55) \ - item(TxAlterCdcStream, 56) \ - item(TxAlterCdcStreamAtTable, 57) \ - item(TxDropCdcStream, 58) \ - item(TxDropCdcStreamAtTable, 59) \ + item(TxCreateCdcStream, 54) \ + item(TxCreateCdcStreamAtTable, 55) \ + item(TxAlterCdcStream, 56) \ + item(TxAlterCdcStreamAtTable, 57) \ + item(TxDropCdcStream, 58) \ + item(TxDropCdcStreamAtTable, 59) \ item(TxMoveTable, 60) \ item(TxMoveTableIndex, 61) \ item(TxCreateSequence, 62) \ item(TxAlterSequence, 63) \ item(TxDropSequence, 64) \ - item(TxCreateReplication, 65) \ - item(TxAlterReplication, 66) \ - item(TxDropReplication, 67) \ + item(TxCreateReplication, 65) \ + item(TxAlterReplication, 66) \ + item(TxDropReplication, 67) \ // TX_STATE_TYPE_ENUM @@ -163,7 +163,7 @@ struct TTxState { item(ProposedDeleteParts, 130, "") \ item(TransferData, 131, "") \ item(NotifyPartitioningChanged, 132, "") \ - item(Aborting, 133, "") \ + item(Aborting, 133, "") \ item(DeleteExternalShards, 134, "") \ item(DeletePrivateShards, 135, "") \ item(WaitShadowPathPublication, 136, "") \ @@ -201,28 +201,28 @@ struct TTxState { {} }; - struct TShardStatus { - bool Success = false; - TString Error; - ui64 BytesProcessed = 0; - ui64 RowsProcessed = 0; - - TShardStatus() = default; - - explicit TShardStatus(bool success, const TString& error, ui64 bytes, ui64 rows) - : Success(success) - , Error(error) - , BytesProcessed(bytes) - , RowsProcessed(rows) - { - } - - explicit TShardStatus(const TString& error) - : TShardStatus(false, error, 0, 0) - { - } - }; - + struct TShardStatus { + bool Success = false; + TString Error; + ui64 BytesProcessed = 0; + ui64 RowsProcessed = 0; + + TShardStatus() = default; + + explicit TShardStatus(bool success, const TString& error, ui64 bytes, ui64 rows) + : Success(success) + , Error(error) + , BytesProcessed(bytes) + , RowsProcessed(rows) + { + } + + explicit TShardStatus(const TString& error) + : TShardStatus(false, error, 0, 0) + { + } + }; + // persist - TxInFlight: ETxType TxType = TxInvalid; TPathId TargetPathId = InvalidPathId; // path (dir or table) being modified @@ -240,10 +240,10 @@ struct TTxState { std::shared_ptr<NKikimrTxDataShard::TSplitMergeDescription> SplitDescription; bool TxShardsListFinalized = false; TTxId BuildIndexId; - // fields below used for backup/restore - bool Cancel = false; - THashMap<TShardIdx, TShardStatus> ShardStatuses; - ui64 DataTotalSize = 0; + // fields below used for backup/restore + bool Cancel = false; + THashMap<TShardIdx, TShardStatus> ShardStatuses; + ui64 DataTotalSize = 0; TMessageSeqNo SchemeOpSeqNo; // For SS -> DS propose events @@ -305,12 +305,12 @@ struct TTxState { case TxCreateRtmrVolume: case TxCreateTableIndex: case TxFillIndex: - case TxCreateCdcStream: + case TxCreateCdcStream: case TxCreateSequence: - case TxCreateReplication: + case TxCreateReplication: return true; case TxInitializeBuildIndex: //this is more like alter - case TxCreateCdcStreamAtTable: + case TxCreateCdcStreamAtTable: return false; case TxCreateLockForIndexBuild: //this is more like alter case TxDropLock: //this is more like alter @@ -330,10 +330,10 @@ struct TTxState { case TxRmDir: case TxFinalizeBuildIndex: case TxDropTableIndexAtMainTable: - case TxDropCdcStream: - case TxDropCdcStreamAtTable: + case TxDropCdcStream: + case TxDropCdcStreamAtTable: case TxDropSequence: - case TxDropReplication: + case TxDropReplication: return false; case TxAlterPQGroup: case TxAlterTable: @@ -343,7 +343,7 @@ struct TTxState { case TxSplitTablePartition: case TxMergeTablePartition: case TxBackup: - case TxRestore: + case TxRestore: case TxAlterBlockStoreVolume: case TxAssignBlockStoreVolume: case TxAlterFileStore: @@ -355,10 +355,10 @@ struct TTxState { case TxAlterUserAttributes: case TxAlterTableIndex: case TxAlterSolomonVolume: - case TxAlterCdcStream: - case TxAlterCdcStreamAtTable: + case TxAlterCdcStream: + case TxAlterCdcStreamAtTable: case TxAlterSequence: - case TxAlterReplication: + case TxAlterReplication: return false; case TxMoveTable: case TxMoveTableIndex: @@ -384,9 +384,9 @@ struct TTxState { case TxDropTableIndex: case TxDropSolomonVolume: case TxRmDir: - case TxDropCdcStream: + case TxDropCdcStream: case TxDropSequence: - case TxDropReplication: + case TxDropReplication: return true; case TxMkDir: case TxCreateTable: @@ -403,16 +403,16 @@ struct TTxState { case TxCreateRtmrVolume: case TxCreateTableIndex: case TxFillIndex: - case TxCreateCdcStream: - case TxCreateCdcStreamAtTable: + case TxCreateCdcStream: + case TxCreateCdcStreamAtTable: case TxCreateSequence: - case TxCreateReplication: + case TxCreateReplication: case TxInitializeBuildIndex: case TxCreateLockForIndexBuild: case TxDropLock: case TxFinalizeBuildIndex: case TxDropTableIndexAtMainTable: // just increments schemaversion at main table - case TxDropCdcStreamAtTable: + case TxDropCdcStreamAtTable: return false; case TxAlterPQGroup: case TxAlterTable: @@ -422,7 +422,7 @@ struct TTxState { case TxSplitTablePartition: case TxMergeTablePartition: case TxBackup: - case TxRestore: + case TxRestore: case TxAlterBlockStoreVolume: case TxAssignBlockStoreVolume: case TxAlterFileStore: @@ -434,10 +434,10 @@ struct TTxState { case TxAlterUserAttributes: case TxAlterTableIndex: case TxAlterSolomonVolume: - case TxAlterCdcStream: - case TxAlterCdcStreamAtTable: + case TxAlterCdcStream: + case TxAlterCdcStreamAtTable: case TxAlterSequence: - case TxAlterReplication: + case TxAlterReplication: return false; case TxMoveTable: case TxMoveTableIndex: @@ -463,9 +463,9 @@ struct TTxState { case TxDropSolomonVolume: case TxSplitTablePartition: case TxMergeTablePartition: - case TxDropCdcStream: + case TxDropCdcStream: case TxDropSequence: - case TxDropReplication: + case TxDropReplication: return true; case TxDropTableIndex: case TxRmDir: @@ -485,15 +485,15 @@ struct TTxState { case TxCreateSolomonVolume: case TxCreateRtmrVolume: case TxCreateTableIndex: - case TxCreateCdcStream: - case TxCreateCdcStreamAtTable: + case TxCreateCdcStream: + case TxCreateCdcStreamAtTable: case TxCreateSequence: - case TxCreateReplication: + case TxCreateReplication: case TxInitializeBuildIndex: case TxCreateLockForIndexBuild: case TxDropLock: case TxDropTableIndexAtMainTable: - case TxDropCdcStreamAtTable: + case TxDropCdcStreamAtTable: return false; case TxAlterPQGroup: case TxAlterTable: @@ -501,7 +501,7 @@ struct TTxState { case TxAlterOlapTable: case TxModifyACL: case TxBackup: - case TxRestore: + case TxRestore: case TxAlterBlockStoreVolume: case TxAssignBlockStoreVolume: case TxAlterFileStore: @@ -514,12 +514,12 @@ struct TTxState { case TxFillIndex: case TxAlterTableIndex: case TxAlterSolomonVolume: - case TxAlterCdcStream: - case TxAlterCdcStreamAtTable: + case TxAlterCdcStream: + case TxAlterCdcStreamAtTable: case TxMoveTable: case TxMoveTableIndex: case TxAlterSequence: - case TxAlterReplication: + case TxAlterReplication: return false; case TxInvalid: Y_VERIFY_DEBUG("UNREACHEBLE"); diff --git a/ydb/core/tx/schemeshard/schemeshard_types.h b/ydb/core/tx/schemeshard/schemeshard_types.h index 097474b5e07..a60df6317da 100644 --- a/ydb/core/tx/schemeshard/schemeshard_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_types.h @@ -11,22 +11,22 @@ namespace NKikimr { namespace NSchemeShard { struct TSchemeLimits { - // path + // path ui64 MaxDepth = 32; ui64 MaxPaths = 200*1000; ui64 MaxChildrenInDir = 100*1000; - ui64 MaxAclBytesSize = 10 << 10; + ui64 MaxAclBytesSize = 10 << 10; ui64 MaxPathElementLength = 255; TString ExtraPathSymbolsAllowed = "!\"#$%&'()*+,-.:;<=>?@[\\]^_`{|}~"; - - // table - ui64 MaxTableColumns = 200; - ui64 MaxTableColumnNameLength = 255; - ui64 MaxTableKeyColumns = 20; - ui64 MaxTableIndices = 20; + + // table + ui64 MaxTableColumns = 200; + ui64 MaxTableColumnNameLength = 255; + ui64 MaxTableKeyColumns = 20; + ui64 MaxTableIndices = 20; ui64 MaxShards = 200*1000; // In each database ui64 MaxShardsInPath = 35*1000; // In each path in database - ui64 MaxConsistentCopyTargets = 1000; + ui64 MaxConsistentCopyTargets = 1000; // pq group ui64 MaxPQPartitions = 1000000; diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.h b/ydb/core/tx/schemeshard/schemeshard_utils.h index 948b097ef7e..de7d17f316f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.h +++ b/ydb/core/tx/schemeshard/schemeshard_utils.h @@ -45,12 +45,12 @@ inline bool IsValidColumnName(const TString& name) { inline NKikimrSchemeOp::TModifyScheme TransactionTemplate(const TString& workingDir, NKikimrSchemeOp::EOperationType type) { NKikimrSchemeOp::TModifyScheme tx; - tx.SetWorkingDir(workingDir); - tx.SetOperationType(type); - - return tx; -} - + tx.SetWorkingDir(workingDir); + tx.SetOperationType(type); + + return tx; +} + TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram& histogram, const TConstArrayRef<NScheme::TTypeId>& keyColumnTypes); diff --git a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp index b8d598c4ba9..58cc395905c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp @@ -57,15 +57,15 @@ bool ValidateUnit(const TColumn& column, NKikimrSchemeOp::TTTLSettings::EUnit un bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, const THashMap<ui32, TTableInfo::TColumn>& sourceColumns, const THashMap<ui32, TTableInfo::TColumn>& alterColumns, - const THashMap<TString, ui32>& colName2Id, - const TSubDomainInfo& subDomain, TString& errStr) + const THashMap<TString, ui32>& colName2Id, + const TSubDomainInfo& subDomain, TString& errStr) { using TTtlProto = NKikimrSchemeOp::TTTLSettings; switch (ttl.GetStatusCase()) { case TTtlProto::kEnabled: { - const auto& enabled = ttl.GetEnabled(); - const TString colName = enabled.GetColumnName(); + const auto& enabled = ttl.GetEnabled(); + const TString colName = enabled.GetColumnName(); auto it = colName2Id.find(colName); if (it == colName2Id.end()) { @@ -88,18 +88,18 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, return false; } - const auto unit = enabled.GetColumnUnit(); + const auto unit = enabled.GetColumnUnit(); if (!ValidateUnit(*column, unit, errStr)) { return false; } - + if (enabled.HasSysSettings()) { const auto& sys = enabled.GetSysSettings(); if (TDuration::FromValue(sys.GetRunInterval()) < subDomain.GetTtlMinRunInterval()) { errStr = Sprintf("TTL run interval cannot be less than limit: %" PRIu64, subDomain.GetTtlMinRunInterval().Seconds()); return false; - } - } + } + } break; } diff --git a/ydb/core/tx/schemeshard/schemeshard_xxport__get.h b/ydb/core/tx/schemeshard/schemeshard_xxport__get.h index 859a4a5d41c..fb22a6458d6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_xxport__get.h +++ b/ydb/core/tx/schemeshard/schemeshard_xxport__get.h @@ -1,49 +1,49 @@ #pragma once -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_impl.h" - +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_impl.h" + #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -namespace NKikimr { + +namespace NKikimr { namespace NSchemeShard { - -template <typename TInfo, typename TEvRequest, typename TEvResponse> + +template <typename TInfo, typename TEvRequest, typename TEvResponse> struct TSchemeShard::TXxport::TTxGet: public TSchemeShard::TXxport::TTxBase { - using TTxGetBase = TTxGet<TInfo, TEvRequest, TEvResponse>; - - typename TEvRequest::TPtr Request; - - explicit TTxGet(TSelf* self, typename TEvRequest::TPtr& ev) - : TTxBase(self) - , Request(ev) - { - } - - bool DoExecuteImpl(const THashMap<ui64, typename TInfo::TPtr>& container, TTransactionContext&, const TActorContext&) { - const auto& request = Request->Get()->Record; - - auto response = MakeHolder<TEvResponse>(); - auto& entry = *response->Record.MutableResponse()->MutableEntry(); - - auto it = container.find(request.GetRequest().GetId()); - if (it == container.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { - entry.SetStatus(Ydb::StatusIds::NOT_FOUND); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - Self->FromXxportInfo(entry, it->second); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxGet - + using TTxGetBase = TTxGet<TInfo, TEvRequest, TEvResponse>; + + typename TEvRequest::TPtr Request; + + explicit TTxGet(TSelf* self, typename TEvRequest::TPtr& ev) + : TTxBase(self) + , Request(ev) + { + } + + bool DoExecuteImpl(const THashMap<ui64, typename TInfo::TPtr>& container, TTransactionContext&, const TActorContext&) { + const auto& request = Request->Get()->Record; + + auto response = MakeHolder<TEvResponse>(); + auto& entry = *response->Record.MutableResponse()->MutableEntry(); + + auto it = container.find(request.GetRequest().GetId()); + if (it == container.end() || !IsSameDomain(it->second, request.GetDatabaseName())) { + entry.SetStatus(Ydb::StatusIds::NOT_FOUND); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + Self->FromXxportInfo(entry, it->second); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxGet + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_xxport__list.h b/ydb/core/tx/schemeshard/schemeshard_xxport__list.h index 6446286c1d3..2af8c7948a8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_xxport__list.h +++ b/ydb/core/tx/schemeshard/schemeshard_xxport__list.h @@ -1,91 +1,91 @@ #pragma once -#include "schemeshard_xxport__tx_base.h" -#include "schemeshard_impl.h" - -#include <util/string/cast.h> - -namespace NKikimr { +#include "schemeshard_xxport__tx_base.h" +#include "schemeshard_impl.h" + +#include <util/string/cast.h> + +namespace NKikimr { namespace NSchemeShard { - -template <typename TInfo, typename TEvRequest, typename TEvResponse, typename TDerived> + +template <typename TInfo, typename TEvRequest, typename TEvResponse, typename TDerived> struct TSchemeShard::TXxport::TTxList: public TSchemeShard::TXxport::TTxBase { - using TTxListBase = TTxList<TInfo, TEvRequest, TEvResponse, TDerived>; - - static constexpr ui64 DefaultPageSize = 10; - static constexpr ui64 MinPageSize = 1; - static constexpr ui64 MaxPageSize = 100; - static constexpr ui64 DefaultPage = 1; - - typename TEvRequest::TPtr Request; - - explicit TTxList(TSelf* self, typename TEvRequest::TPtr& ev) - : TTxBase(self) - , Request(ev) - { - } - - bool DoExecuteImpl(const THashMap<ui64, typename TInfo::TPtr>& container, TTransactionContext&, const TActorContext&) { - const auto& record = Request->Get()->Record; - const auto& request = record.GetRequest(); - - auto response = MakeHolder<TEvResponse>(); - auto& resp = *response->Record.MutableResponse(); - - const TPathId domainPathId = DomainPathId(record.GetDatabaseName()); - const ui64 pageSize = Min(request.GetPageSize() ? Max(request.GetPageSize(), MinPageSize) : DefaultPageSize, MaxPageSize); - - ui64 page = DefaultPage; - if (request.GetPageToken() && !TryFromString(request.GetPageToken(), page)) { - resp.SetStatus(Ydb::StatusIds::BAD_REQUEST); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - page = Max(page, DefaultPage); - - typename TInfo::EKind kind; - if (!TDerived::TryParseKind(request.GetKind(), kind)) { - resp.SetStatus(Ydb::StatusIds::BAD_REQUEST); - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - resp.SetStatus(Ydb::StatusIds::SUCCESS); - - auto it = container.begin(); - ui64 skip = (page - 1) * pageSize; - while (it != container.end() && skip) { - if (IsSameDomain(it->second, domainPathId) && it->second->Kind == kind) { - --skip; - } - ++it; - } - - ui64 size = 0; - while (it != container.end() && size < pageSize) { - if (IsSameDomain(it->second, domainPathId) && it->second->Kind == kind) { - Self->FromXxportInfo(*resp.MutableEntries()->Add(), it->second); - ++size; - } - ++it; - } - - if (it == container.end()) { - resp.SetNextPageToken("1"); - } else { - resp.SetNextPageToken(ToString(page + 1)); - } - - Send(Request->Sender, std::move(response), 0, Request->Cookie); - return true; - } - - void DoComplete(const TActorContext&) override { - } - -}; // TTxList - + using TTxListBase = TTxList<TInfo, TEvRequest, TEvResponse, TDerived>; + + static constexpr ui64 DefaultPageSize = 10; + static constexpr ui64 MinPageSize = 1; + static constexpr ui64 MaxPageSize = 100; + static constexpr ui64 DefaultPage = 1; + + typename TEvRequest::TPtr Request; + + explicit TTxList(TSelf* self, typename TEvRequest::TPtr& ev) + : TTxBase(self) + , Request(ev) + { + } + + bool DoExecuteImpl(const THashMap<ui64, typename TInfo::TPtr>& container, TTransactionContext&, const TActorContext&) { + const auto& record = Request->Get()->Record; + const auto& request = record.GetRequest(); + + auto response = MakeHolder<TEvResponse>(); + auto& resp = *response->Record.MutableResponse(); + + const TPathId domainPathId = DomainPathId(record.GetDatabaseName()); + const ui64 pageSize = Min(request.GetPageSize() ? Max(request.GetPageSize(), MinPageSize) : DefaultPageSize, MaxPageSize); + + ui64 page = DefaultPage; + if (request.GetPageToken() && !TryFromString(request.GetPageToken(), page)) { + resp.SetStatus(Ydb::StatusIds::BAD_REQUEST); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + page = Max(page, DefaultPage); + + typename TInfo::EKind kind; + if (!TDerived::TryParseKind(request.GetKind(), kind)) { + resp.SetStatus(Ydb::StatusIds::BAD_REQUEST); + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + resp.SetStatus(Ydb::StatusIds::SUCCESS); + + auto it = container.begin(); + ui64 skip = (page - 1) * pageSize; + while (it != container.end() && skip) { + if (IsSameDomain(it->second, domainPathId) && it->second->Kind == kind) { + --skip; + } + ++it; + } + + ui64 size = 0; + while (it != container.end() && size < pageSize) { + if (IsSameDomain(it->second, domainPathId) && it->second->Kind == kind) { + Self->FromXxportInfo(*resp.MutableEntries()->Add(), it->second); + ++size; + } + ++it; + } + + if (it == container.end()) { + resp.SetNextPageToken("1"); + } else { + resp.SetNextPageToken(ToString(page + 1)); + } + + Send(Request->Sender, std::move(response), 0, Request->Cookie); + return true; + } + + void DoComplete(const TActorContext&) override { + } + +}; // TTxList + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_xxport__tx_base.h b/ydb/core/tx/schemeshard/schemeshard_xxport__tx_base.h index 66989a991c1..84b46815c2c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_xxport__tx_base.h +++ b/ydb/core/tx/schemeshard/schemeshard_xxport__tx_base.h @@ -1,91 +1,91 @@ -#pragma once - -#include "schemeshard_impl.h" - -#include <util/generic/ptr.h> -#include <util/generic/vector.h> - -namespace NKikimr { +#pragma once + +#include "schemeshard_impl.h" + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> + +namespace NKikimr { namespace NSchemeShard { - + class TSchemeShard::TXxport::TTxBase: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> { - TVector<THolder<IEventHandle>> SendOnComplete; - -protected: - explicit TTxBase(TSelf* self) - : TBase(self) - { - } - - virtual ~TTxBase() = default; - + TVector<THolder<IEventHandle>> SendOnComplete; + +protected: + explicit TTxBase(TSelf* self) + : TBase(self) + { + } + + virtual ~TTxBase() = default; + void Send(const TActorId& recipient, IEventBase* ev, ui32 flags = 0, ui64 cookie = 0) { - SendOnComplete.emplace_back(new IEventHandle(recipient, Self->SelfId(), ev, flags, cookie)); - } - - template <typename TEvent> + SendOnComplete.emplace_back(new IEventHandle(recipient, Self->SelfId(), ev, flags, cookie)); + } + + template <typename TEvent> void Send(const TActorId& recipient, THolder<TEvent> ev, ui32 flags = 0, ui64 cookie = 0) { - return Send(recipient, static_cast<IEventBase*>(ev.Release()), flags, cookie); - } - - TPathId DomainPathId(const TString& dbName) const { - const TPath domainPath = TPath::Resolve(dbName, Self); - if (domainPath.IsResolved()) { - return domainPath.Base()->PathId; - } - - return TPathId(); - } - - template <typename TInfoPtr> - static bool IsSameDomain(const TInfoPtr info, const TPathId& domainPathId) { - if (!domainPathId) { - return true; - } - - return info->DomainPathId == domainPathId; - } - - template <typename TInfoPtr> - bool IsSameDomain(const TInfoPtr info, const TString& dbName) const { - return IsSameDomain(info, DomainPathId(dbName)); - } - - template <typename TInfoPtr> - void SendNotificationsIfFinished(TInfoPtr info, bool force = false) { - if (!info->IsFinished() && !force) { + return Send(recipient, static_cast<IEventBase*>(ev.Release()), flags, cookie); + } + + TPathId DomainPathId(const TString& dbName) const { + const TPath domainPath = TPath::Resolve(dbName, Self); + if (domainPath.IsResolved()) { + return domainPath.Base()->PathId; + } + + return TPathId(); + } + + template <typename TInfoPtr> + static bool IsSameDomain(const TInfoPtr info, const TPathId& domainPathId) { + if (!domainPathId) { + return true; + } + + return info->DomainPathId == domainPathId; + } + + template <typename TInfoPtr> + bool IsSameDomain(const TInfoPtr info, const TString& dbName) const { + return IsSameDomain(info, DomainPathId(dbName)); + } + + template <typename TInfoPtr> + void SendNotificationsIfFinished(TInfoPtr info, bool force = false) { + if (!info->IsFinished() && !force) { return; } - LOG_TRACE_S(TlsActivationContext->AsActorContext(), NKikimrServices::FLAT_TX_SCHEMESHARD, - "SendNotifications: " - << ": id# " << info->Id - << ", subscribers count# " << info->Subscribers.size()); + LOG_TRACE_S(TlsActivationContext->AsActorContext(), NKikimrServices::FLAT_TX_SCHEMESHARD, + "SendNotifications: " + << ": id# " << info->Id + << ", subscribers count# " << info->Subscribers.size()); TSet<TActorId> toAnswer; - toAnswer.swap(info->Subscribers); + toAnswer.swap(info->Subscribers); for (auto& actorId: toAnswer) { Send(actorId, new TEvSchemeShard::TEvNotifyTxCompletionResult(info->Id)); } } -public: - virtual bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) = 0; - virtual void DoComplete(const TActorContext& ctx) = 0; - - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { - return DoExecute(txc, ctx); - } - - void Complete(const TActorContext& ctx) override { - DoComplete(ctx); - - for (auto& ev : SendOnComplete) { - ctx.Send(std::move(ev)); - } - } - -}; // TTxBase - +public: + virtual bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) = 0; + virtual void DoComplete(const TActorContext& ctx) = 0; + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + return DoExecute(txc, ctx); + } + + void Complete(const TActorContext& ctx) override { + DoComplete(ctx); + + for (auto& ev : SendOnComplete) { + ctx.Send(std::move(ev)); + } + } + +}; // TTxBase + } // NSchemeShard -} // NKikimr +} // NKikimr diff --git a/ydb/core/tx/schemeshard/ut_async_index.cpp b/ydb/core/tx/schemeshard/ut_async_index.cpp index 83ccb24bc79..a2fc2d39741 100644 --- a/ydb/core/tx/schemeshard/ut_async_index.cpp +++ b/ydb/core/tx/schemeshard/ut_async_index.cpp @@ -1,78 +1,78 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/testlib/tablet_helpers.h> - -using namespace NKikimr; + +using namespace NKikimr; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(TAsyncIndexTests) { - void CreateTable(bool enableAsyncIndexes) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableAsyncIndexes(enableAsyncIndexes)); - ui64 txId = 100; - - const auto status = enableAsyncIndexes +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TAsyncIndexTests) { + void CreateTable(bool enableAsyncIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableAsyncIndexes(enableAsyncIndexes)); + ui64 txId = 100; + + const auto status = enableAsyncIndexes ? NKikimrScheme::StatusAccepted : NKikimrScheme::StatusPreconditionFailed; - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "indexed" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndex" - KeyColumnNames: ["indexed"] - Type: EIndexTypeGlobalAsync - } - )", {status}); - - if (enableAsyncIndexes) { - env.TestWaitNotification(runtime, txId); - } - } - - Y_UNIT_TEST(CreateTableShouldSucceed) { - CreateTable(true); - } - - Y_UNIT_TEST(CreateTableShouldFail) { - CreateTable(false); - } - - void OnlineBuild(bool enableAsyncIndexes) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableAsyncIndexes(enableAsyncIndexes)); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "indexed" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - const auto status = enableAsyncIndexes - ? Ydb::StatusIds::SUCCESS - : Ydb::StatusIds::UNSUPPORTED; - + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndex" + KeyColumnNames: ["indexed"] + Type: EIndexTypeGlobalAsync + } + )", {status}); + + if (enableAsyncIndexes) { + env.TestWaitNotification(runtime, txId); + } + } + + Y_UNIT_TEST(CreateTableShouldSucceed) { + CreateTable(true); + } + + Y_UNIT_TEST(CreateTableShouldFail) { + CreateTable(false); + } + + void OnlineBuild(bool enableAsyncIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableAsyncIndexes(enableAsyncIndexes)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + const auto status = enableAsyncIndexes + ? Ydb::StatusIds::SUCCESS + : Ydb::StatusIds::UNSUPPORTED; + TestBuilIndex(runtime, ++txId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", TBuildIndexConfig{ "UserDefinedIndex", NKikimrSchemeOp::EIndexTypeGlobalAsync, {"indexed"}, {} - }, status); - - if (enableAsyncIndexes) { - env.TestWaitNotification(runtime, txId); - } - } - - Y_UNIT_TEST(OnlineBuildShouldSucceed) { - OnlineBuild(true); - } - - Y_UNIT_TEST(OnlineBuildShouldFail) { - OnlineBuild(false); - } -} + }, status); + + if (enableAsyncIndexes) { + env.TestWaitNotification(runtime, txId); + } + } + + Y_UNIT_TEST(OnlineBuildShouldSucceed) { + OnlineBuild(true); + } + + Y_UNIT_TEST(OnlineBuildShouldFail) { + OnlineBuild(false); + } +} diff --git a/ydb/core/tx/schemeshard/ut_async_index/ya.make b/ydb/core/tx/schemeshard/ut_async_index/ya.make index aea78796e3e..fdb326de6a5 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/ya.make +++ b/ydb/core/tx/schemeshard/ut_async_index/ya.make @@ -1,32 +1,32 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(60) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( ydb/core/testlib ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_async_index.cpp -) - +) + +SRCS( + ut_async_index.cpp +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index 7ea96344e35..a76dcf9d5bb 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -76,7 +76,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::PathVersionEqual(3)}); - TestRmDir(runtime, ++txId, "/MyRoot/DirA", "SubDirA", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/MyRoot/DirA", "SubDirA", {NKikimrScheme::StatusNameConflict}); TestRmDir(runtime, ++txId, "/MyRoot/DirA/SubDirA", "DirB"); env.TestWaitNotification(runtime, txId); TestRmDir(runtime, ++txId, "/MyRoot/DirA", "SubDirA"); @@ -106,7 +106,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestRmDir(runtime, ++txId, "/MyRoot", "DirA"); TestRmDir(runtime, ++txId, "/MyRoot", "DirB"); env.TestWaitNotification(runtime, {txId-1, txId}); - TestRmDir(runtime, ++txId, "/", "MyRoot", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/", "MyRoot", {NKikimrScheme::StatusNameConflict}); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot"), @@ -115,63 +115,63 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::ShardsInsideDomain(0)}); } - using TRuntimeTxFn = std::function<void(TTestBasicRuntime&, ui64)>; - - void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - createFn(runtime, ++txId); - env.TestWaitNotification(runtime, txId); - - dropFn(runtime, ++txId); - dropFn(runtime, ++txId); - TestModificationResult(runtime, txId - 1); - + using TRuntimeTxFn = std::function<void(TTestBasicRuntime&, ui64)>; + + void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + createFn(runtime, ++txId); + env.TestWaitNotification(runtime, txId); + + dropFn(runtime, ++txId); + dropFn(runtime, ++txId); + TestModificationResult(runtime, txId - 1); + auto ev = runtime.GrabEdgeEvent<TEvSchemeShard::TEvModifySchemeTransactionResult>(); - UNIT_ASSERT(ev); - - const auto& record = ev->Record; - UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); + UNIT_ASSERT(ev); + + const auto& record = ev->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusMultipleModifications); - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 1); - - env.TestWaitNotification(runtime, txId - 1); - TestDescribeResult(DescribePath(runtime, path), { - NLs::PathNotExist - }); - } - - Y_UNIT_TEST(RmDirTwice) { - auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) { - TestMkDir(runtime, txId, "/MyRoot", "Dir"); - }; - - auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) { - AsyncRmDir(runtime, txId, "/MyRoot", "Dir"); - }; - - DropTwice("/MyRoot/Dir", createFn, dropFn); - } - - Y_UNIT_TEST(DropTableTwice) { - auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) { - TestCreateTable(runtime, txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - }; - - auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) { - AsyncDropTable(runtime, txId, "/MyRoot", "Table"); - }; - - DropTwice("/MyRoot/Table", createFn, dropFn); - } - + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 1); + + env.TestWaitNotification(runtime, txId - 1); + TestDescribeResult(DescribePath(runtime, path), { + NLs::PathNotExist + }); + } + + Y_UNIT_TEST(RmDirTwice) { + auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) { + TestMkDir(runtime, txId, "/MyRoot", "Dir"); + }; + + auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) { + AsyncRmDir(runtime, txId, "/MyRoot", "Dir"); + }; + + DropTwice("/MyRoot/Dir", createFn, dropFn); + } + + Y_UNIT_TEST(DropTableTwice) { + auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) { + TestCreateTable(runtime, txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + }; + + auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) { + AsyncDropTable(runtime, txId, "/MyRoot", "Table"); + }; + + DropTwice("/MyRoot/Table", createFn, dropFn); + } + Y_UNIT_TEST(CacheEffectiveACL) { TEffectiveACL firstACL; @@ -686,10 +686,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { "KeyColumnNames: [\"RowId\", \"key1\", \"key2\"]" ); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", - "Name: \"x/y\"" - "Columns { Name: \"key\" Type: \"Uint32\"}" - "KeyColumnNames: [\"key\"]" - ); + "Name: \"x/y\"" + "Columns { Name: \"key\" Type: \"Uint32\"}" + "KeyColumnNames: [\"key\"]" + ); AsyncCreateTable(runtime, ++txId, "/MyRoot/DirA", "Name: \"Table3\"" "Columns { Name: \"RowId\" Type: \"Yson\"}" @@ -792,7 +792,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::ChildrenCount(1)}); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::PathVersionEqual(13), - NLs::ChildrenCount(5)}); + NLs::ChildrenCount(5)}); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/Table1"), {NLs::PathExist}); @@ -1074,7 +1074,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::ChildrenCount(2) }); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1103,7 +1103,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::PathsInsideDomain(17), NLs::ShardsInsideDomain(10)}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/dst1" DstPath: "/MyRoot/DirA/seconddst1" @@ -1152,14 +1152,14 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::CreatedAt(txId-1) }); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"}), NLs::IndexDataColumns({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/dst2/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), {NLs::Finished}); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/seconddst1"), @@ -1178,7 +1178,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::CreatedAt(txId) }); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/seconddst1" DstPath: "/MyRoot/DirA/thirddst1" @@ -1250,7 +1250,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::Finished}); AsyncMkDir(runtime, ++txId, "/MyRoot", "DirB"); - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirB/await1" @@ -1264,7 +1264,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestModificationResult(runtime, txId, NKikimrScheme::StatusMultipleModifications); env.TestWaitNotification(runtime, {txId-1, txId}); - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirB/await1" @@ -1342,7 +1342,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::ChildrenCount(3) }); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1353,10 +1353,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {NKikimrScheme::StatusInvalidParameter}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( )", {NKikimrScheme::StatusInvalidParameter}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/src1" @@ -1367,7 +1367,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {NKikimrScheme::StatusSchemeError}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA" DstPath: "/MyRoot/DirA/dst1" @@ -1378,7 +1378,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {NKikimrScheme::StatusNameConflict}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1389,7 +1389,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {NKikimrScheme::StatusPathDoesNotExist}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1400,14 +1400,14 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } )", {NKikimrScheme::StatusPathDoesNotExist}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/src2" } )", {NKikimrScheme::StatusSchemeError}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1423,7 +1423,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { env.TestWaitNotification(runtime, txId); RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1{not*allowed$symbols!" @@ -1440,7 +1440,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::PathExist, NLs::Finished}); - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1454,7 +1454,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { //Still in fly checks //retry, must never happen with the same txId, but check that behavior just in case - AsyncConsistentCopyTables(runtime, copyingTx, "/", R"( + AsyncConsistentCopyTables(runtime, copyingTx, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1466,7 +1466,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); //retry with different body, just in case - AsyncConsistentCopyTables(runtime, copyingTx, "/", R"( + AsyncConsistentCopyTables(runtime, copyingTx, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src2" DstPath: "/MyRoot/DirA/dst1" @@ -1477,7 +1477,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { })"); //simultaneous - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1487,7 +1487,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { DstPath: "/MyRoot/DirA/dst2" })"); - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src3" DstPath: "/MyRoot/DirA/dst1" @@ -1505,7 +1505,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::Finished, NLs::ChildrenCount(5)}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/x1" @@ -1547,7 +1547,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestCopyTable(runtime, ++txId, "/MyRoot/DirA", "NotOk", "/MyRoot/DirA/WithEnabledExternalBlobs", NKikimrScheme::StatusPreconditionFailed); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/IsOk" @@ -1598,7 +1598,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestCopyTable(runtime, ++txId, "/MyRoot/DirA", "NotOk", "/MyRoot/DirA/IndexedTableWithEnabledExternalBlobs", NKikimrScheme::StatusPreconditionFailed); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/IsOk" @@ -1627,7 +1627,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestCopyTable(runtime, ++txId, "/MyRoot/DirA", "NotOk", "/MyRoot/DirA/IndexedTableWithEnabledExternalBlobs", NKikimrScheme::StatusPreconditionFailed); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/IsOk" @@ -1670,7 +1670,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::ChildrenCount(2) }); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1688,7 +1688,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestModificationResult(runtime, txId, NKikimrScheme::StatusAccepted); env.TestWaitNotification(runtime, {txId, txId-1}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1703,7 +1703,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { //simonteniously AsyncDropTable(runtime, ++txId, "/MyRoot/DirA", "dst1"); AsyncDropTable(runtime, ++txId, "/MyRoot/DirA", "dst2"); - AsyncConsistentCopyTables(runtime, ++txId, "/", R"( + AsyncConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirA/src1" DstPath: "/MyRoot/DirA/dst1" @@ -1755,7 +1755,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::NotFinished, NLs::PathVersionEqual(1), NLs::IndexesCount(0)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1"), {NLs::NotFinished}); TestModificationResult(runtime, txId-1, NKikimrScheme::StatusAccepted); TestModificationResult(runtime, txId, NKikimrScheme::StatusAccepted); @@ -1769,37 +1769,37 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(4)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue1/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValues"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValues"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0", "value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValues/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValues/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"}), NLs::IndexDataColumns({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), {NLs::Finished}); TestDropTable(runtime, ++txId, "/MyRoot/DirA", "Table1"); @@ -1852,29 +1852,29 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(3)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue1/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"}), NLs::IndexDataColumns({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirA/copy/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), {NLs::Finished}); } @@ -2245,15 +2245,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestModificationResult(runtime, txId-2, NKikimrScheme::StatusAccepted); TestModificationResult(runtime, txId-1, NKikimrScheme::StatusAccepted); - { + { auto ev = runtime.GrabEdgeEvent<TEvSchemeShard::TEvModifySchemeTransactionResult>(); - UNIT_ASSERT(ev); - const auto& record = ev->Record; - UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); + UNIT_ASSERT(ev); + const auto& record = ev->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusMultipleModifications); - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 2); - } - + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 2); + } + env.TestWaitNotification(runtime, {txId-1, txId-2}); TestDropTable(runtime, ++txId, "/MyRoot/DirA", "Table1", {TEvSchemeShard::EStatus::StatusPathDoesNotExist}); @@ -3395,169 +3395,169 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { }; }; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), {descrChecker(1)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), {descrChecker(1)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable2", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable2", true), {descrChecker(2)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable3", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable3", true), {descrChecker(1)}); } Y_UNIT_TEST(CopyTableOmitFollowers) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - // create src table - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "Value" Type: "Utf8"} - KeyColumnNames: ["key"] - PartitionConfig { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + // create src table + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "Value" Type: "Utf8"} + KeyColumnNames: ["key"] + PartitionConfig { FollowerCount: 1 - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { - NLs::Finished, - NLs::IsTable, + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { + NLs::Finished, + NLs::IsTable, NLs::FollowerCount(1) - }); - - // simple copy table - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "CopyTable" - CopyFromTable: "/MyRoot/Table" + }); + + // simple copy table + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "CopyTable" + CopyFromTable: "/MyRoot/Table" OmitFollowers: true - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), { - NLs::Finished, - NLs::IsTable, + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), { + NLs::Finished, + NLs::IsTable, NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), NLs::FollowerGroups({NKikimrHive::TFollowerGroup()}) - }); - - // consistent copy table - TestConsistentCopyTables(runtime, ++txId, "/", R"( - CopyTableDescriptions { - SrcPath: "/MyRoot/Table" - DstPath: "/MyRoot/ConsistentCopyTable" + }); + + // consistent copy table + TestConsistentCopyTables(runtime, ++txId, "/", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/ConsistentCopyTable" OmitFollowers: true - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/ConsistentCopyTable", true), { - NLs::Finished, - NLs::IsTable, + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/ConsistentCopyTable", true), { + NLs::Finished, + NLs::IsTable, NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), NLs::FollowerGroups({NKikimrHive::TFollowerGroup()}) - }); - } - - Y_UNIT_TEST(CopyTableForBackup) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - // create src table - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "value" Type: "Utf8"} - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { - NLs::IsBackupTable(false), - }); - - // simple copy table - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "CopyTable" - CopyFromTable: "/MyRoot/Table" - IsBackup: true - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), { - NLs::IsBackupTable(true), - }); - - // consistent copy table - TestConsistentCopyTables(runtime, ++txId, "/", R"( - CopyTableDescriptions { - SrcPath: "/MyRoot/Table" - DstPath: "/MyRoot/ConsistentCopyTable" - IsBackup: true - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/ConsistentCopyTable", true), { - NLs::IsBackupTable(true), - }); - - // cannot create new table with 'IsBackup' - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TableForBackup" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "value" Type: "Utf8"} - KeyColumnNames: ["key"] - IsBackup: true + }); + } + + Y_UNIT_TEST(CopyTableForBackup) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + // create src table + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { + NLs::IsBackupTable(false), + }); + + // simple copy table + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "CopyTable" + CopyFromTable: "/MyRoot/Table" + IsBackup: true + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/CopyTable", true), { + NLs::IsBackupTable(true), + }); + + // consistent copy table + TestConsistentCopyTables(runtime, ++txId, "/", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/ConsistentCopyTable" + IsBackup: true + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/ConsistentCopyTable", true), { + NLs::IsBackupTable(true), + }); + + // cannot create new table with 'IsBackup' + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TableForBackup" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + IsBackup: true )", {NKikimrScheme::StatusInvalidParameter}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "TableForBackup" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "value" Type: "Utf8"} - KeyColumnNames: ["key"] - IsBackup: true - } - IndexDescription { - Name: "UserDefinedIndexByValue" - KeyColumnNames: ["value"] - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableForBackup" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + IsBackup: true + } + IndexDescription { + Name: "UserDefinedIndexByValue" + KeyColumnNames: ["value"] + } )", {NKikimrScheme::StatusInvalidParameter}); - - // cannot add 'IsBackup' property to existent table - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - IsBackup: true + + // cannot add 'IsBackup' property to existent table + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + IsBackup: true )", {NKikimrScheme::StatusInvalidParameter}); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - IsBackup: true - DropColumns { Name: "value" } + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + IsBackup: true + DropColumns { Name: "value" } )", {NKikimrScheme::StatusInvalidParameter}); - - // cannot remove 'IsBackup' property from existent table - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "CopyTable" - IsBackup: false + + // cannot remove 'IsBackup' property from existent table + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "CopyTable" + IsBackup: false )", {NKikimrScheme::StatusInvalidParameter}); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "CopyTable" - IsBackup: false - DropColumns { Name: "value" } + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "CopyTable" + IsBackup: false + DropColumns { Name: "value" } )", {NKikimrScheme::StatusInvalidParameter}); - } - + } + Y_UNIT_TEST(AlterTableAndConcurrentSplit) { //+ TTestBasicRuntime runtime; TTestEnv env(runtime); @@ -3664,7 +3664,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestModificationResult(runtime, txId, NKikimrScheme::StatusMultipleModifications); env.TestWaitNotification(runtime, {txId, txId-1, txId-2}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), {NLs::PartitionCount(2)}); NTabletFlatScheme::TSchemeChanges scheme; @@ -4311,7 +4311,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { UNIT_ASSERT_VALUES_EQUAL(tableDescription.GetColumns(1).GetFamily(), 0); } - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), {NLs::IsTable, [] (const NKikimrScheme::TEvDescribeSchemeResult& record) { Cerr << record.DebugString() << Endl; @@ -4543,34 +4543,34 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { }; Cerr << "Checking Table1" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), {descrChecker()}); Cerr << "Checking Table2" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), {descrChecker()}); Cerr << "Checking Table3" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), {descrChecker(1)}); TActorId sender = runtime.AllocateEdgeActor(); RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); Cerr << "Checking Table1" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), {descrChecker()}); Cerr << "Checking Table2" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), {descrChecker()}); Cerr << "Checking Table3" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), {descrChecker(1)}); Cerr << "Checking Table4" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table4", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table4", true), {descrChecker(1)}); // Copy table of table with families should succeed @@ -4647,15 +4647,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )", {NKikimrScheme::StatusInvalidParameter}); Cerr << "Checking Table1" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), {descrChecker()}); Cerr << "Checking Table2" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table2", true), {descrChecker()}); Cerr << "Checking Table3" << Endl; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table3", true), {descrChecker()}); // /MyRoot/Table1 @@ -4832,12 +4832,12 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { UNIT_ASSERT_VALUES_EQUAL(tableDescription.GetColumns(0).GetFamily(), 0); }; - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1", true), {NLs::ColumnFamiliesCount(1), NLs::ColumnFamiliesHas(0, "ExtBlobsOnHDD"), descrChecker}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1Copy", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1Copy", true), {descrChecker, NLs::ColumnFamiliesHas(0, "ExtBlobsOnHDD"), NLs::ColumnFamiliesCount(1)}); @@ -4981,7 +4981,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } Cerr << "Checking schema for Table1" << Endl; - TestLs(runtime, "/MyRoot/Table1", true, checker); + TestLs(runtime, "/MyRoot/Table1", true, checker); }; // Creating a table with alt family on a different data storage should be OK @@ -5289,7 +5289,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot"), {NLs::PathsInsideDomain(1), NLs::ShardsInsideDomain(1)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5324,7 +5324,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot"), {NLs::PathsInsideDomain(1), NLs::ShardsInsideDomain(1)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(2), NLs::CrossDataCenterFollowerCount(0), @@ -5342,7 +5342,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(1), NLs::CrossDataCenterFollowerCount(0), @@ -5364,7 +5364,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(1), NLs::CrossDataCenterFollowerCount(0), @@ -5382,7 +5382,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(1), NLs::CrossDataCenterFollowerCount(0), @@ -5400,7 +5400,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5418,7 +5418,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(1), @@ -5435,7 +5435,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5452,7 +5452,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5469,7 +5469,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )"); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(2), @@ -5485,7 +5485,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { )", {NKikimrScheme::StatusInvalidParameter}); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(2), @@ -5508,7 +5508,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5530,7 +5530,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5552,7 +5552,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5574,7 +5574,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5600,7 +5600,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5624,7 +5624,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5648,7 +5648,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5679,7 +5679,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NKikimrHive::TFollowerGroup control; control.SetFollowerCount(1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5707,7 +5707,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { control.SetAllowLeaderPromotion(true); control.SetRequireAllDataCenters(true); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), { NLs::FollowerCount(0), NLs::CrossDataCenterFollowerCount(0), @@ -5979,7 +5979,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::Finished}); } - Y_UNIT_TEST(AlterPersQueueGroup) { //+ + Y_UNIT_TEST(AlterPersQueueGroup) { //+ TTestBasicRuntime runtime; TTestEnv env(runtime); ui64 txId = 1000; @@ -6088,154 +6088,154 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::ShardsInsideDomain(41)}); } - Y_UNIT_TEST(CreatePersQueueGroupWithKeySchema) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - // Partition count and partition boundaries mismatch - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 1 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - } + Y_UNIT_TEST(CreatePersQueueGroupWithKeySchema) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + // Partition count and partition boundaries mismatch + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 1 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + } )", {NKikimrScheme::StatusInvalidParameter}); - - // Missing key schema - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 2 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - } + + // Missing key schema + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 2 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + } )", {NKikimrScheme::StatusInvalidParameter}); - - // Invalid partition boundary - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 2 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - Tuple { Optional { Uint32: 2000 } } - } + + // Invalid partition boundary + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 2 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + Tuple { Optional { Uint32: 2000 } } + } )", {NKikimrScheme::StatusSchemeError}); - - // Invalid type - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 2 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 0 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - } + + // Invalid type + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 2 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 0 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + } )", {NKikimrScheme::StatusSchemeError}); - - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup1" - TotalGroupCount: 1 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - } - )"); - env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup1", true), { - NLs::CheckPartCount("PQGroup1", 1, 1, 1, 1), - }); - - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup2" - TotalGroupCount: 2 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - } - )"); - env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup2", true), { - NLs::CheckPartCount("PQGroup2", 2, 1, 2, 2), - }); - - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup3" - TotalGroupCount: 2 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - PartitionKeySchema { Name: "key2" TypeId: 2 } - } - PartitionBoundaries { - Tuple { Optional { Uint32: 1000 } } - } - )"); - env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup3", true), { - NLs::CheckPartCount("PQGroup3", 2, 1, 2, 2), - }); - } - - Y_UNIT_TEST(AlterPersQueueGroupWithKeySchema) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 1 - PartitionPerTablet: 1 - PQTabletConfig { - PartitionConfig { LifetimeSeconds: 10 } - PartitionKeySchema { Name: "key1" TypeId: 2 } - } - )"); - env.TestWaitNotification(runtime, txId); - - // change partition count - TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - TotalGroupCount: 2 + + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup1" + TotalGroupCount: 1 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + } + )"); + env.TestWaitNotification(runtime, txId); + TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup1", true), { + NLs::CheckPartCount("PQGroup1", 1, 1, 1, 1), + }); + + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup2" + TotalGroupCount: 2 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + } + )"); + env.TestWaitNotification(runtime, txId); + TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup2", true), { + NLs::CheckPartCount("PQGroup2", 2, 1, 2, 2), + }); + + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup3" + TotalGroupCount: 2 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + PartitionKeySchema { Name: "key2" TypeId: 2 } + } + PartitionBoundaries { + Tuple { Optional { Uint32: 1000 } } + } + )"); + env.TestWaitNotification(runtime, txId); + TestDescribeResult(DescribePath(runtime, "/MyRoot/PQGroup3", true), { + NLs::CheckPartCount("PQGroup3", 2, 1, 2, 2), + }); + } + + Y_UNIT_TEST(AlterPersQueueGroupWithKeySchema) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreatePQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 1 + PartitionPerTablet: 1 + PQTabletConfig { + PartitionConfig { LifetimeSeconds: 10 } + PartitionKeySchema { Name: "key1" TypeId: 2 } + } + )"); + env.TestWaitNotification(runtime, txId); + + // change partition count + TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + TotalGroupCount: 2 )", {NKikimrScheme::StatusInvalidParameter}); - - // add partitions - TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - PartitionsToAdd { PartitionId: 1 GroupId: 1 } + + // add partitions + TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + PartitionsToAdd { PartitionId: 1 GroupId: 1 } )", {NKikimrScheme::StatusInvalidParameter}); - - // change key schema - TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( - Name: "PQGroup" - PQTabletConfig { - PartitionKeySchema { Name: "key2" TypeId: 2 } - } + + // change key schema + TestAlterPQGroup(runtime, ++txId, "/MyRoot", R"( + Name: "PQGroup" + PQTabletConfig { + PartitionKeySchema { Name: "key2" TypeId: 2 } + } )", {NKikimrScheme::StatusInvalidParameter}); - } - + } + Y_UNIT_TEST(DropTable) { //+ TTestBasicRuntime runtime; TTestEnv env(runtime); @@ -6265,7 +6265,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops/Table", true), {NLs::Finished}); - TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); TestDropTable(runtime, ++txId, "/MyRoot/Ops", "Table"); env.TestWaitNotification(runtime, txId); @@ -6285,7 +6285,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops/PartTable", true), {NLs::Finished}); - TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); TestDropTable(runtime, ++txId, "/MyRoot/Ops", "PartTable"); env.TestWaitNotification(runtime, txId); @@ -6361,7 +6361,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops"), {NLs::Finished}); - TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); TestDropPQGroup(runtime, ++txId, "/MyRoot/Ops", "DropMeBaby"); env.TestWaitNotification(runtime, txId); @@ -6405,7 +6405,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops/DropMeBaby", true), {NLs::Finished}); - TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); + TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusNameConflict}); TestDropTable(runtime, ++txId, "/MyRoot/Ops", "DropMeBaby"); env.TestWaitNotification(runtime, txId); @@ -6445,7 +6445,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops"), {NLs::PathNotExist}); - TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusPathDoesNotExist}); + TestRmDir(runtime, ++txId, "/MyRoot", "Ops", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitTabletDeletion(runtime, xrange(TTestTxConfig::FakeHiveTablets, TTestTxConfig::FakeHiveTablets+40)); } @@ -7252,7 +7252,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::PathsInsideDomain(1), NLs::ShardsInsideDomain(1)}); } - Y_UNIT_TEST(AlterBlockStoreVolume) { //+ + Y_UNIT_TEST(AlterBlockStoreVolume) { //+ TTestBasicRuntime runtime; TTestEnv env(runtime, /* nchannels */ 6); ui64 txId = 100; @@ -8674,105 +8674,105 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::PathsInsideDomain(0), NLs::ShardsInsideDomain(0)}); } - void UpdateChannelsBindingSolomon(bool allow) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().AllowUpdateChannelsBindingOfSolomonPartitions(allow)); - ui64 txId = 100; - - auto check = [&](const TString& path, ui64 shards, const TVector<THashMap<TString, ui32>>& expectedChannels) { + void UpdateChannelsBindingSolomon(bool allow) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().AllowUpdateChannelsBindingOfSolomonPartitions(allow)); + ui64 txId = 100; + + auto check = [&](const TString& path, ui64 shards, const TVector<THashMap<TString, ui32>>& expectedChannels) { NKikimrSchemeOp::TDescribeOptions opts; - opts.SetReturnChannelsBinding(true); - - TestDescribeResult(DescribePath(runtime, path, opts), { - NLs::Finished, - NLs::ShardsInsideDomain(shards), + opts.SetReturnChannelsBinding(true); + + TestDescribeResult(DescribePath(runtime, path, opts), { + NLs::Finished, + NLs::ShardsInsideDomain(shards), [&expectedChannels] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& desc = record.GetPathDescription().GetSolomonDescription(); - - UNIT_ASSERT_VALUES_EQUAL(expectedChannels.size(), desc.PartitionsSize()); - - for (size_t i = 0; i < desc.PartitionsSize(); ++i) { - const auto& partition = desc.GetPartitions(i); - - THashMap<TString, ui32> channels; - for (const auto& channel : partition.GetBoundChannels()) { - auto it = channels.find(channel.GetStoragePoolName()); - if (it == channels.end()) { - it = channels.emplace(channel.GetStoragePoolName(), 0).first; - } - ++it->second; - } - - UNIT_ASSERT_VALUES_EQUAL(expectedChannels.at(i).size(), channels.size()); - - for (const auto& [name, count] : expectedChannels.at(i)) { - UNIT_ASSERT_C(channels.contains(name), "Cannot find channel: " << name); - UNIT_ASSERT_VALUES_EQUAL(channels.at(name), count); - } - } - } - }); - }; - - TestCreateSolomon(runtime, ++txId, "/MyRoot", R"( - Name: "Solomon" - PartitionCount: 1 - ChannelProfileId: 2 - )"); - - env.TestWaitNotification(runtime, txId); - check("/MyRoot/Solomon", 1, {{{"pool-1", 4}}}); - - // case 1: empty alter - TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( - Name: "Solomon" - ChannelProfileId: 3 + const auto& desc = record.GetPathDescription().GetSolomonDescription(); + + UNIT_ASSERT_VALUES_EQUAL(expectedChannels.size(), desc.PartitionsSize()); + + for (size_t i = 0; i < desc.PartitionsSize(); ++i) { + const auto& partition = desc.GetPartitions(i); + + THashMap<TString, ui32> channels; + for (const auto& channel : partition.GetBoundChannels()) { + auto it = channels.find(channel.GetStoragePoolName()); + if (it == channels.end()) { + it = channels.emplace(channel.GetStoragePoolName(), 0).first; + } + ++it->second; + } + + UNIT_ASSERT_VALUES_EQUAL(expectedChannels.at(i).size(), channels.size()); + + for (const auto& [name, count] : expectedChannels.at(i)) { + UNIT_ASSERT_C(channels.contains(name), "Cannot find channel: " << name); + UNIT_ASSERT_VALUES_EQUAL(channels.at(name), count); + } + } + } + }); + }; + + TestCreateSolomon(runtime, ++txId, "/MyRoot", R"( + Name: "Solomon" + PartitionCount: 1 + ChannelProfileId: 2 + )"); + + env.TestWaitNotification(runtime, txId); + check("/MyRoot/Solomon", 1, {{{"pool-1", 4}}}); + + // case 1: empty alter + TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( + Name: "Solomon" + ChannelProfileId: 3 )", {NKikimrScheme::StatusInvalidParameter}); - - // case 2: add partition, do not update channels binding - TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( - Name: "Solomon" - ChannelProfileId: 3 - PartitionCount: 2 - )"); - - env.TestWaitNotification(runtime, txId); - check("/MyRoot/Solomon", 2, {{{"pool-1", 4}}, {{"pool-2", 4}}}); - - // case 3: just update channels binding - TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( - Name: "Solomon" - ChannelProfileId: 3 - UpdateChannelsBinding: true + + // case 2: add partition, do not update channels binding + TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( + Name: "Solomon" + ChannelProfileId: 3 + PartitionCount: 2 + )"); + + env.TestWaitNotification(runtime, txId); + check("/MyRoot/Solomon", 2, {{{"pool-1", 4}}, {{"pool-2", 4}}}); + + // case 3: just update channels binding + TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( + Name: "Solomon" + ChannelProfileId: 3 + UpdateChannelsBinding: true )", {allow ? NKikimrScheme::StatusAccepted : NKikimrScheme::StatusPreconditionFailed}); - - if (!allow) { - return; - } - - env.TestWaitNotification(runtime, txId); - check("/MyRoot/Solomon", 2, {{{"pool-2", 4}}, {{"pool-2", 4}}}); - - // case 4: add partition & update channels binding - TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( - Name: "Solomon" - ChannelProfileId: 2 - PartitionCount: 3 - UpdateChannelsBinding: true - )"); - - env.TestWaitNotification(runtime, txId); - check("/MyRoot/Solomon", 3, {{{"pool-1", 4}}, {{"pool-1", 4}}, {{"pool-1", 4}}}); - } - - Y_UNIT_TEST(UpdateChannelsBindingSolomonShouldNotUpdate) { - UpdateChannelsBindingSolomon(false); - } - - Y_UNIT_TEST(UpdateChannelsBindingSolomonShouldUpdate) { - UpdateChannelsBindingSolomon(true); - } - + + if (!allow) { + return; + } + + env.TestWaitNotification(runtime, txId); + check("/MyRoot/Solomon", 2, {{{"pool-2", 4}}, {{"pool-2", 4}}}); + + // case 4: add partition & update channels binding + TestAlterSolomon(runtime, ++txId, "/MyRoot", R"( + Name: "Solomon" + ChannelProfileId: 2 + PartitionCount: 3 + UpdateChannelsBinding: true + )"); + + env.TestWaitNotification(runtime, txId); + check("/MyRoot/Solomon", 3, {{{"pool-1", 4}}, {{"pool-1", 4}}, {{"pool-1", 4}}}); + } + + Y_UNIT_TEST(UpdateChannelsBindingSolomonShouldNotUpdate) { + UpdateChannelsBindingSolomon(false); + } + + Y_UNIT_TEST(UpdateChannelsBindingSolomonShouldUpdate) { + UpdateChannelsBindingSolomon(true); + } + Y_UNIT_TEST(RejectAlterSolomon) { //+ TTestBasicRuntime runtime; TTestEnv env(runtime); @@ -8927,7 +8927,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot"), {NLs::PathExist}); - auto pathVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1"), + auto pathVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1"), {NLs::PathExist, NLs::PathVersionEqual(3)}); @@ -8943,53 +8943,53 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::PathsInsideDomain(0), NLs::ShardsInsideDomainOneOf({0, 1})}); } - - Y_UNIT_TEST(CreateWithIntermediateDirs) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 123; - - TestMkDir(runtime, ++txId, "/MyRoot", "DirA/DirB"); + + Y_UNIT_TEST(CreateWithIntermediateDirs) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 123; + + TestMkDir(runtime, ++txId, "/MyRoot", "DirA/DirB"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/DirB"), {NLs::Finished}); - - TestMkDir(runtime, ++txId, "/MyRoot/DirA", "DirB/DirC"); + + TestMkDir(runtime, ++txId, "/MyRoot/DirA", "DirB/DirC"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/DirB/DirC"), {NLs::Finished}); - - TestCreateTable(runtime, ++txId, "/MyRoot", - R"( Name: "Tables/Table1" - Columns { Name: "RowId" Type: "Uint64" } - KeyColumnNames: ["RowId"] - )"); + + TestCreateTable(runtime, ++txId, "/MyRoot", + R"( Name: "Tables/Table1" + Columns { Name: "RowId" Type: "Uint64" } + KeyColumnNames: ["RowId"] + )"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/Tables"), {NLs::Finished}); TestDescribeResult(DescribePath(runtime, "/MyRoot/Tables/Table1"), {NLs::Finished, NLs::IsTable}); - + TestCreatePQGroup(runtime, ++txId, "/MyRoot", - R"( Name: "PQ/Topic1" - TotalGroupCount: 2 - PartitionPerTablet: 2 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10}} - )"); + R"( Name: "PQ/Topic1" + TotalGroupCount: 2 + PartitionPerTablet: 2 + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10}} + )"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/PQ"), {NLs::Finished}); TestDescribeResult(DescribePath(runtime, "/MyRoot/PQ/Topic1"), {NLs::Finished}); - + TestMkDir(runtime, ++txId, "/MyRoot", "DirA/DirB", {NKikimrScheme::StatusAlreadyExists}); TestMkDir(runtime, ++txId, "/MyRoot/DirC", "DirA/DirB", {NKikimrScheme::StatusPathDoesNotExist}); TestMkDir(runtime, ++txId, "/MyRoot/DirA", "/DirD/DirE", {NKikimrScheme::StatusSchemeError}); TestMkDir(runtime, ++txId, "/MyRoot/DirA", "DirD/DirE/", {NKikimrScheme::StatusSchemeError}); TestMkDir(runtime, ++txId, "/MyRoot/Tables/Table1", "DirA/DirB", {NKikimrScheme::StatusPathIsNotDirectory}); - - TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); + + TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/x/y/z"), {NLs::Finished, @@ -9001,27 +9001,27 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { {NLs::Finished, NLs::CreatedAt(txId)}); - TestRmDir(runtime, ++txId, "/MyRoot/x/y", "z"); + TestRmDir(runtime, ++txId, "/MyRoot/x/y", "z"); env.TestWaitNotification(runtime, txId); - TestRmDir(runtime, ++txId, "/MyRoot/x", "y"); + TestRmDir(runtime, ++txId, "/MyRoot/x", "y"); env.TestWaitNotification(runtime, txId); - TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); + TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); env.TestWaitNotification(runtime, txId); - - TestRmDir(runtime, ++txId, "/MyRoot/x/y", "z"); + + TestRmDir(runtime, ++txId, "/MyRoot/x/y", "z"); env.TestWaitNotification(runtime, txId); - TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); + TestMkDir(runtime, ++txId, "/MyRoot", "x/y/z"); env.TestWaitNotification(runtime, txId); - - TestCreateTable(runtime, ++txId, "/MyRoot", - R"( Name: "WrongTables/Table1" - Columns { Name: "RowId" Type: "Uint64" } - KeyColumnNames: ["WrongRowId"] + + TestCreateTable(runtime, ++txId, "/MyRoot", + R"( Name: "WrongTables/Table1" + Columns { Name: "RowId" Type: "Uint64" } + KeyColumnNames: ["WrongRowId"] )", {NKikimrScheme::StatusSchemeError}); TestDescribeResult(DescribePath(runtime, "/MyRoot/WrongTables"), {NLs::PathNotExist}); - } + } Y_UNIT_TEST(AlterTableAndAfterSplit) { //+ TTestBasicRuntime runtime; @@ -9349,17 +9349,17 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { "Columns { Name: \"Key\" Type: \"Uint64\"} " "Columns { Name: \"Value\" Type: \"Uint64\"} " "KeyColumnNames: [\"Key\"]", - {NKikimrScheme::StatusAccepted}, + {NKikimrScheme::StatusAccepted}, AlterUserAttrs({{"__document_api_version", "1"}})); env.TestWaitNotification(runtime, txId); // Changing document api version is not allowed - TestUserAttrs(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusInvalidParameter}, - AlterUserAttrs({{"__document_api_version", "2"}}), { }); + TestUserAttrs(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusInvalidParameter}, + AlterUserAttrs({{"__document_api_version", "2"}}), { }); // Removing document api version is not allowed - TestUserAttrs(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusInvalidParameter}, - AlterUserAttrs({{"__document_api_version", ""}}), { }); + TestUserAttrs(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusInvalidParameter}, + AlterUserAttrs({{"__document_api_version", ""}}), { }); // Create a new table without an attribute TestCreateTable(runtime, ++txId, "/MyRoot", @@ -9370,8 +9370,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { env.TestWaitNotification(runtime, txId); // Adding document api version is not allowed - TestUserAttrs(runtime, ++txId, "/MyRoot", "TableWoDocumentApi", {NKikimrScheme::StatusInvalidParameter}, - AlterUserAttrs({{"__document_api_version", "1"}}), { }); + TestUserAttrs(runtime, ++txId, "/MyRoot", "TableWoDocumentApi", {NKikimrScheme::StatusInvalidParameter}, + AlterUserAttrs({{"__document_api_version", "1"}}), { }); // Creating other objects (e.g. directories) with document api version is not allowed TestMkDir(runtime, ++txId, "/MyRoot", "DirA", @@ -9734,10 +9734,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestCopyTable(runtime, ++txId, "/MyRoot", "Copy1", "/MyRoot/Table1"); env.TestWaitNotification(runtime, txId); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table1" - IndexName: "Sync" - )"); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table1" + IndexName: "Sync" + )"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot"), @@ -9757,7 +9757,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { NLs::CheckColumns("Copy1", {"key", "value"}, {}, {"key"}), NLs::IndexesCount(1)}); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/Table1" DstPath: "/MyRoot/Copy2" @@ -9837,7 +9837,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { env.TestWaitNotification(runtime, 103); - TestConsistentCopyTables(runtime, ++txId, "/", R"( + TestConsistentCopyTables(runtime, ++txId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/Table" DstPath: "/MyRoot/copy" diff --git a/ydb/core/tx/schemeshard/ut_base_reboots.cpp b/ydb/core/tx/schemeshard/ut_base_reboots.cpp index b47546c8bd4..05fef3170c3 100644 --- a/ydb/core/tx/schemeshard/ut_base_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_base_reboots.cpp @@ -30,7 +30,7 @@ Y_UNIT_TEST_SUITE(TTablesWithReboots) { { TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table", true), {NLs::Finished, NLs::IsTable, NLs::PartitionCount(2)}); @@ -65,7 +65,7 @@ Y_UNIT_TEST_SUITE(TTablesWithReboots) { TestDescribeResult(DescribePath(runtime, "/MyRoot"), {NLs::ChildrenCount(3)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/NewTable", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/NewTable", true), {NLs::Finished, NLs::IsTable, NLs::PartitionCount(2), @@ -438,7 +438,7 @@ Y_UNIT_TEST_SUITE(TTablesWithReboots) { { TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "MyRoot/Table", true), {NLs::Finished, NLs::FreezeStateEqual(NKikimrSchemeOp::EFreezeState::Freeze)}); } @@ -451,7 +451,7 @@ Y_UNIT_TEST_SUITE(TTablesWithReboots) { { TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "MyRoot/Table", true), + TestDescribeResult(DescribePath(runtime, "MyRoot/Table", true), {NLs::Finished, NLs::FreezeStateEqual(NKikimrSchemeOp::EFreezeState::Unfreeze)}); } diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream.cpp b/ydb/core/tx/schemeshard/ut_cdc_stream.cpp index 6ba81be78e0..5090fe7df6a 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/ut_cdc_stream.cpp @@ -1,335 +1,335 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - -using namespace NKikimr; + +using namespace NKikimr; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(TCdcStreamTests) { - Y_UNIT_TEST(Basic) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathExist}); - - TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - Disable {} - )"); - env.TestWaitNotification(runtime, txId); - - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - )"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); - } - - Y_UNIT_TEST(CreateStream) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestCreateCdcStream(runtime, ++txId, "/", R"( - TableName: "MyRoot" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TCdcStreamTests) { + Y_UNIT_TEST(Basic) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathExist}); + + TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + Disable {} + )"); + env.TestWaitNotification(runtime, txId); + + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); + } + + Y_UNIT_TEST(CreateStream) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateCdcStream(runtime, ++txId, "/", R"( + TableName: "MyRoot" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } )", {NKikimrScheme::StatusNameConflict}); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "UnknownTable" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "UnknownTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } )", {NKikimrScheme::StatusPathDoesNotExist}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "indexed" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "Index" - KeyColumnNames: ["indexed"] - } - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Index" - Mode: ECdcStreamModeKeysOnly - } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Index" + KeyColumnNames: ["indexed"] + } + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Index" + Mode: ECdcStreamModeKeysOnly + } )", {NKikimrScheme::StatusNameConflict}); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot/Table/Index", R"( - TableName: "indexImplTable" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } + + TestCreateCdcStream(runtime, ++txId, "/MyRoot/Table/Index", R"( + TableName: "indexImplTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } )", {NKikimrScheme::StatusNameConflict}); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - } + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + } )", {NKikimrScheme::StatusInvalidParameter}); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDropTable(runtime, ++txId, "/MyRoot", "Table"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDropTable(runtime, ++txId, "/MyRoot", "Table"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } )", {NKikimrScheme::StatusPathDoesNotExist}); - } - - Y_UNIT_TEST(AlterStream) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestAlterCdcStream(runtime, ++txId, "/", R"( - TableName: "MyRoot" - StreamName: "Stream" - Disable {} + } + + Y_UNIT_TEST(AlterStream) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestAlterCdcStream(runtime, ++txId, "/", R"( + TableName: "MyRoot" + StreamName: "Stream" + Disable {} )", {NKikimrScheme::StatusNameConflict}); - - TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "UnknownTable" - StreamName: "Stream" - Disable {} + + TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "UnknownTable" + StreamName: "Stream" + Disable {} )", {NKikimrScheme::StatusPathDoesNotExist}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "indexed" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "Index" - KeyColumnNames: ["indexed"] - } - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Index" - Disable {} + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Index" + KeyColumnNames: ["indexed"] + } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Index" + Disable {} )", {NKikimrScheme::StatusNameConflict}); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" )", {NKikimrScheme::StatusInvalidParameter}); - } - - Y_UNIT_TEST(DropStream) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestDropCdcStream(runtime, ++txId, "/", R"( - TableName: "MyRoot" - StreamName: "Stream" - )", {NKikimrScheme::StatusNameConflict}); - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "UnknownTable" - StreamName: "Stream" - )", {NKikimrScheme::StatusPathDoesNotExist}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "indexed" Type: "Uint64" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "Index" - KeyColumnNames: ["indexed"] - } - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Index" - )", {NKikimrScheme::StatusNameConflict}); - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "UnknownStream" - )", {NKikimrScheme::StatusPathDoesNotExist}); - - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - )"); - env.TestWaitNotification(runtime, txId); - - TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - )", {NKikimrScheme::StatusPathDoesNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - - // drop table + index + stream - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDropTable(runtime, ++txId, "/MyRoot", "Table"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); - - // drop table + stream - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDropTable(runtime, ++txId, "/MyRoot", "Table"); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); - } - - Y_UNIT_TEST(AlterStreamImplShouldFail) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterPQGroup(runtime, ++txId, "/MyRoot/Table/Stream", R"( - Name: "streamImpl" - PQTabletConfig: { PartitionConfig { LifetimeSeconds: 3600 } } - )", {NKikimrScheme::StatusNameConflict}); - } - - Y_UNIT_TEST(DropStreamImplShouldFail) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - env.TestWaitNotification(runtime, txId); - - TestDropPQGroup(runtime, ++txId, "/MyRoot/Table/Stream", "streamImpl", {NKikimrScheme::StatusNameConflict}); - } - -} // TCdcStreamTests + } + + Y_UNIT_TEST(DropStream) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestDropCdcStream(runtime, ++txId, "/", R"( + TableName: "MyRoot" + StreamName: "Stream" + )", {NKikimrScheme::StatusNameConflict}); + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "UnknownTable" + StreamName: "Stream" + )", {NKikimrScheme::StatusPathDoesNotExist}); + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Index" + KeyColumnNames: ["indexed"] + } + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Index" + )", {NKikimrScheme::StatusNameConflict}); + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "UnknownStream" + )", {NKikimrScheme::StatusPathDoesNotExist}); + + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )"); + env.TestWaitNotification(runtime, txId); + + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )", {NKikimrScheme::StatusPathDoesNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + + // drop table + index + stream + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDropTable(runtime, ++txId, "/MyRoot", "Table"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); + + // drop table + stream + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDropTable(runtime, ++txId, "/MyRoot", "Table"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream/streamImpl"), {NLs::PathNotExist}); + } + + Y_UNIT_TEST(AlterStreamImplShouldFail) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterPQGroup(runtime, ++txId, "/MyRoot/Table/Stream", R"( + Name: "streamImpl" + PQTabletConfig: { PartitionConfig { LifetimeSeconds: 3600 } } + )", {NKikimrScheme::StatusNameConflict}); + } + + Y_UNIT_TEST(DropStreamImplShouldFail) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDropPQGroup(runtime, ++txId, "/MyRoot/Table/Stream", "streamImpl", {NKikimrScheme::StatusNameConflict}); + } + +} // TCdcStreamTests diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream/ya.make b/ydb/core/tx/schemeshard/ut_cdc_stream/ya.make index ea9d3738b4a..770e485be1e 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream/ya.make +++ b/ydb/core/tx/schemeshard/ut_cdc_stream/ya.make @@ -1,34 +1,34 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(2) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(2) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( ydb/core/testlib ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_cdc_stream.cpp -) - +) + +SRCS( + ut_cdc_stream.cpp +) + YQL_LAST_ABI_VERSION() - + REQUIREMENTS(ram:11) -END() +END() diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp index 595b3b98411..376544ace83 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp @@ -1,165 +1,165 @@ -#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(TCdcStreamWithRebootsTests) { - Y_UNIT_TEST(CreateStream) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); - }); - } - - Y_UNIT_TEST(AlterStream) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - auto request = AlterCdcStreamRequest(++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - Disable {} - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusMultipleModifications, - }); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - }); - } - - Y_UNIT_TEST(DropStream) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - TestDropCdcStream(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - }); - } - - Y_UNIT_TEST(CreateDropRecreate) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - { - auto request = CreateCdcStreamRequest(++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusAlreadyExists, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); - - { - auto request = DropCdcStreamRequest(++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamName: "Stream" - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); - - { - auto request = CreateCdcStreamRequest(++t.TxId, "/MyRoot", R"( - TableName: "Table" - StreamDescription { - Name: "Stream" - Mode: ECdcStreamModeKeysOnly - } - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusAlreadyExists, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); - }); - } - -} // TCdcStreamWithRebootsTests +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TCdcStreamWithRebootsTests) { + Y_UNIT_TEST(CreateStream) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); + }); + } + + Y_UNIT_TEST(AlterStream) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + auto request = AlterCdcStreamRequest(++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + Disable {} + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusMultipleModifications, + }); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + }); + } + + Y_UNIT_TEST(DropStream) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropCdcStream(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + }); + } + + Y_UNIT_TEST(CreateDropRecreate) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + { + auto request = CreateCdcStreamRequest(++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); + + { + auto request = DropCdcStreamRequest(++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathNotExist}); + + { + auto request = CreateCdcStreamRequest(++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + } + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), {NLs::PathExist}); + }); + } + +} // TCdcStreamWithRebootsTests diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/ya.make b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/ya.make index 4eaab7ac7d2..c507ee5e628 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/ya.make +++ b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/ya.make @@ -1,31 +1,31 @@ -UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(2) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( - ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_cdc_stream_reboots.cpp -) - -YQL_LAST_ABI_VERSION() - -END() +UNITTEST_FOR(ydb/core/tx/schemeshard) + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(2) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/tx/schemeshard/ut_helpers +) + +SRCS( + ut_cdc_stream_reboots.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ut_compaction.cpp b/ydb/core/tx/schemeshard/ut_compaction.cpp index c30174e9eb8..59f8de87094 100644 --- a/ydb/core/tx/schemeshard/ut_compaction.cpp +++ b/ydb/core/tx/schemeshard/ut_compaction.cpp @@ -70,8 +70,8 @@ void DisableBackgroundCompactionViaRestart( ui64 schemeShard) { // turn on background compaction and restart to apply - runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionForTest(false); - runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionServerlessForTest(false); + runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionForTest(false); + runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionServerlessForTest(false); // little hack to simplify life auto& compactionConfig = runtime.GetAppData().CompactionConfig; @@ -88,8 +88,8 @@ void EnableBackgroundCompactionViaRestart( bool enableServerless) { // turn on background compaction and restart to apply - runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionForTest(true); - runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionServerlessForTest(enableServerless); + runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableBackgroundCompactionServerlessForTest(enableServerless); // little hack to simplify life auto& compactionConfig = runtime.GetAppData().CompactionConfig; @@ -161,7 +161,7 @@ void CheckNoCompactions( ui64 schemeshardId, const TString& path) { - auto description = DescribePrivatePath(runtime, schemeshardId, path, true, true); + auto description = DescribePrivatePath(runtime, schemeshardId, path, true, true); TVector<ui64> shards; for (auto &part : description.GetPathDescription().GetTablePartitions()) shards.push_back(part.GetDatashardId()); @@ -212,7 +212,7 @@ void TestBackgroundCompaction( enableBackgroundCompactionFunc(runtime, env); - auto description = DescribePrivatePath(runtime, "/MyRoot/Simple", true, true); + auto description = DescribePrivatePath(runtime, "/MyRoot/Simple", true, true); TVector<ui64> shards; for (auto &part : description.GetPathDescription().GetTablePartitions()) shards.push_back(part.GetDatashardId()); @@ -308,7 +308,7 @@ ui64 TestServerless( // turn on background compaction EnableBackgroundCompactionViaRestart(runtime, env, schemeshardId, enableServerless); - auto description = DescribePrivatePath(runtime, schemeshardId, "/MyRoot/User/Simple", true, true); + auto description = DescribePrivatePath(runtime, schemeshardId, "/MyRoot/User/Simple", true, true); TVector<ui64> shards; for (auto &part : description.GetPathDescription().GetTablePartitions()) shards.push_back(part.GetDatashardId()); diff --git a/ydb/core/tx/schemeshard/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export.cpp index eaf2347f2db..20c5353befc 100644 --- a/ydb/core/tx/schemeshard/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export.cpp @@ -5,19 +5,19 @@ #include <ydb/core/wrappers/s3_wrapper.h> #include <ydb/core/metering/metering.h> -#include <util/string/builder.h> -#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/string/cast.h> #include <util/string/printf.h> #include <util/system/env.h> using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; -namespace { +namespace { void Run(TTestBasicRuntime& runtime, TTestEnv& env, const TVector<TString>& tables, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") { + const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") { ui64 txId = 100; @@ -55,10 +55,10 @@ namespace { TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", Sprintf(R"( Name: "%s" - ResourcesDomainKey { - SchemeShard: %lu - PathId: 2 - } + ResourcesDomainKey { + SchemeShard: %lu + PathId: 2 + } )", TStringBuf(dbName).RNextTok('/').data(), TTestTxConfig::SchemeShard), attrs); env.TestWaitNotification(runtime, txId); @@ -78,7 +78,7 @@ namespace { Name: "name_User_kind_hdd-2" Kind: "external" } - )", TStringBuf(dbName).RNextTok('/').data())); + )", TStringBuf(dbName).RNextTok('/').data())); env.TestWaitNotification(runtime, txId); } @@ -96,7 +96,7 @@ namespace { runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID); + TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID); env.TestWaitNotification(runtime, txId, schemeshardId); const ui64 exportId = txId; @@ -108,172 +108,172 @@ namespace { TestGetExport(runtime, schemeshardId, exportId, dbName, Ydb::StatusIds::NOT_FOUND); } - using TDelayFunc = std::function<bool(TAutoPtr<IEventHandle>&)>; - - void Cancel(const TVector<TString>& tables, const TString& request, TDelayFunc delayFunc) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - for (const auto& table : tables) { - TestCreateTable(runtime, ++txId, "/MyRoot", table); - env.TestWaitNotification(runtime, txId); - } - - runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - - THolder<IEventHandle> delayed; - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (delayFunc(ev)) { - delayed.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } - return TTestActorRuntime::EEventAction::PROCESS; - }); - - TestExport(runtime, ++txId, "/MyRoot", request); - const ui64 exportId = txId; - - if (!delayed) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return bool(delayed); - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - - TestCancelExport(runtime, ++txId, "/MyRoot", exportId); - runtime.Send(delayed.Release(), 0, true); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::CANCELLED); - - TestForgetExport(runtime, ++txId, "/MyRoot", exportId); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - -} // anonymous - -Y_UNIT_TEST_SUITE(TExportToS3Tests) { - void RunS3(TTestBasicRuntime& runtime, const TVector<TString>& tables, const TString& request) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TTestEnv env(runtime); - Run(runtime, env, tables, Sprintf(request.c_str(), port), Ydb::StatusIds::SUCCESS, "/MyRoot", false); + using TDelayFunc = std::function<bool(TAutoPtr<IEventHandle>&)>; + + void Cancel(const TVector<TString>& tables, const TString& request, TDelayFunc delayFunc) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + for (const auto& table : tables) { + TestCreateTable(runtime, ++txId, "/MyRoot", table); + env.TestWaitNotification(runtime, txId); + } + + runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); + + THolder<IEventHandle> delayed; + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (delayFunc(ev)) { + delayed.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + return TTestActorRuntime::EEventAction::PROCESS; + }); + + TestExport(runtime, ++txId, "/MyRoot", request); + const ui64 exportId = txId; + + if (!delayed) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return bool(delayed); + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + + TestCancelExport(runtime, ++txId, "/MyRoot", exportId); + runtime.Send(delayed.Release(), 0, true); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::CANCELLED); + + TestForgetExport(runtime, ++txId, "/MyRoot", exportId); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + +} // anonymous + +Y_UNIT_TEST_SUITE(TExportToS3Tests) { + void RunS3(TTestBasicRuntime& runtime, const TVector<TString>& tables, const TString& request) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TTestEnv env(runtime); + Run(runtime, env, tables, Sprintf(request.c_str(), port), Ydb::StatusIds::SUCCESS, "/MyRoot", false); + } + + Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { + TTestBasicRuntime runtime; + + RunS3(runtime, { + R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", + }, R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "" + } + } + )"); } - Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { - TTestBasicRuntime runtime; - - RunS3(runtime, { + Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { + TTestBasicRuntime runtime; + + RunS3(runtime, { R"( Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + UniformPartitionsCount: 2 + )", + }, R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "" + } + } + )"); + } + + Y_UNIT_TEST(ShouldSucceedOnManyTables) { + TTestBasicRuntime runtime; + + RunS3(runtime, { + R"( + Name: "Table1" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", + R"( + Name: "Table2" Columns { Name: "key" Type: "Utf8" } Columns { Name: "value" Type: "Utf8" } KeyColumnNames: ["key"] )", }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table1" + destination_prefix: "table1" + } + items { + source_path: "/MyRoot/Table2" + destination_prefix: "table2" + } + } + )"); + } + + void CancelShouldSucceed(TDelayFunc delayFunc) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + Cancel({ + R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", + }, Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP items { source_path: "/MyRoot/Table" - destination_prefix: "" + destination_prefix: "" } } - )"); + )", port), delayFunc); } - Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { - TTestBasicRuntime runtime; - - RunS3(runtime, { - R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - UniformPartitionsCount: 2 - )", - }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/Table" - destination_prefix: "" - } - } - )"); - } - - Y_UNIT_TEST(ShouldSucceedOnManyTables) { - TTestBasicRuntime runtime; - - RunS3(runtime, { - R"( - Name: "Table1" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", - R"( - Name: "Table2" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", - }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/Table1" - destination_prefix: "table1" - } - items { - source_path: "/MyRoot/Table2" - destination_prefix: "table2" - } - } - )"); - } - - void CancelShouldSucceed(TDelayFunc delayFunc) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - Cancel({ - R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", - }, Sprintf(R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/Table" - destination_prefix: "" - } - } - )", port), delayFunc); - } - Y_UNIT_TEST(CancelUponCreatingExportDirShouldSucceed) { CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) { @@ -297,13 +297,13 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) { } void CancelUponTransferringShouldSucceed(const TVector<TString>& tables, const TString& request) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - Cancel(tables, Sprintf(request.c_str(), port), [](TAutoPtr<IEventHandle>& ev) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + Cancel(tables, Sprintf(request.c_str(), port), [](TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) { return false; } @@ -322,12 +322,12 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) { KeyColumnNames: ["key"] )", }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP items { source_path: "/MyRoot/Table" - destination_prefix: "" + destination_prefix: "" } } )"); @@ -343,12 +343,12 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) { UniformPartitionsCount: 2 )", }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP items { source_path: "/MyRoot/Table" - destination_prefix: "" + destination_prefix: "" } } )"); @@ -373,16 +373,16 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) { KeyColumnNames: ["key"] )", }, R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP items { source_path: "/MyRoot/Table1" - destination_prefix: "table1" + destination_prefix: "table1" } items { source_path: "/MyRoot/Table2" - destination_prefix: "table2" + destination_prefix: "table2" } } )"); @@ -469,329 +469,329 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) { TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); } - - void DropCopiesBeforeTransferring(ui32 tablesCount) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - for (ui32 i = 1; i <= tablesCount; ++i) { - TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "Table%d" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", i)); - env.TestWaitNotification(runtime, txId); - } - - runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - - bool dropNotification = false; - THolder<IEventHandle> delayed; - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { + + void DropCopiesBeforeTransferring(ui32 tablesCount) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + for (ui32 i = 1; i <= tablesCount; ++i) { + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "Table%d" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", i)); + env.TestWaitNotification(runtime, txId); + } + + runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); + + bool dropNotification = false; + THolder<IEventHandle> delayed; + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { case TEvSchemeShard::EvModifySchemeTransaction: - break; + break; case TEvSchemeShard::EvNotifyTxCompletionResult: - if (dropNotification) { - delayed.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } - return TTestActorRuntime::EEventAction::PROCESS; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - + if (dropNotification) { + delayed.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + return TTestActorRuntime::EEventAction::PROCESS; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + const auto* msg = ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>(); if (msg->Record.GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpCreateConsistentCopyTables) { - dropNotification = true; - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TStringBuilder items; - for (ui32 i = 1; i <= tablesCount; ++i) { - items << "items {" - << " source_path: \"/MyRoot/Table" << i << "\"" - << " destination_prefix: \"\"" - << " }"; - } - - TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - %s - } - )", port, items.c_str())); - const ui64 exportId = txId; - - if (!delayed) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return bool(delayed); - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - - for (ui32 i = 0; i < tablesCount; ++i) { - TestDropTable(runtime, ++txId, Sprintf("/MyRoot/export-%" PRIu64, exportId), ToString(i)); - env.TestWaitNotification(runtime, txId); - } - - runtime.Send(delayed.Release(), 0, true); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::CANCELLED); - - TestForgetExport(runtime, ++txId, "/MyRoot", exportId); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - - Y_UNIT_TEST(DropCopiesBeforeTransferring1) { - DropCopiesBeforeTransferring(1); - } - - Y_UNIT_TEST(DropCopiesBeforeTransferring2) { - DropCopiesBeforeTransferring(2); - } - - void RebootDuringFinish(bool rejectUploadParts, Ydb::StatusIds::StatusCode expectedStatus) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - auto writeRow = [&](ui64 tabletId, const char* key, const char* value) { - NKikimrMiniKQL::TResult result; - TString error; - NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( - ( - (let key '( '('key (Utf8 '%s) ) ) ) - (let row '( '('value (Utf8 '%s) ) ) ) - (return (AsList (UpdateRow '__user__Table key row) )) - ) - )", key, value), result, error); - - UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - }; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - + dropNotification = true; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TStringBuilder items; + for (ui32 i = 1; i <= tablesCount; ++i) { + items << "items {" + << " source_path: \"/MyRoot/Table" << i << "\"" + << " destination_prefix: \"\"" + << " }"; + } + + TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + %s + } + )", port, items.c_str())); + const ui64 exportId = txId; + + if (!delayed) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return bool(delayed); + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + + for (ui32 i = 0; i < tablesCount; ++i) { + TestDropTable(runtime, ++txId, Sprintf("/MyRoot/export-%" PRIu64, exportId), ToString(i)); + env.TestWaitNotification(runtime, txId); + } + + runtime.Send(delayed.Release(), 0, true); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::CANCELLED); + + TestForgetExport(runtime, ++txId, "/MyRoot", exportId); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + + Y_UNIT_TEST(DropCopiesBeforeTransferring1) { + DropCopiesBeforeTransferring(1); + } + + Y_UNIT_TEST(DropCopiesBeforeTransferring2) { + DropCopiesBeforeTransferring(2); + } + + void RebootDuringFinish(bool rejectUploadParts, Ydb::StatusIds::StatusCode expectedStatus) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + auto writeRow = [&](ui64 tabletId, const char* key, const char* value) { + NKikimrMiniKQL::TResult result; + TString error; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( + ( + (let key '( '('key (Utf8 '%s) ) ) ) + (let row '( '('value (Utf8 '%s) ) ) ) + (return (AsList (UpdateRow '__user__Table key row) )) + ) + )", key, value), result, error); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + writeRow(TTestTxConfig::FakeHiveTablets, "a", "valueA"); writeRow(TTestTxConfig::FakeHiveTablets, "b", "valueB"); - - runtime.SetLogPriority(NKikimrServices::S3_WRAPPER, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - - TMaybe<ui64> backupTxId; - TMaybe<ui64> tabletId; - bool delayed = false; - - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvDataShard::EvProposeTransaction: { - auto& record = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record; - if (record.GetTxKind() != NKikimrTxDataShard::ETransactionKind::TX_KIND_SCHEME) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - NKikimrTxDataShard::TFlatSchemeTransaction schemeTx; - UNIT_ASSERT(schemeTx.ParseFromString(record.GetTxBody())); - - if (schemeTx.HasBackup()) { - backupTxId = record.GetTxId(); - // hijack - schemeTx.MutableBackup()->MutableScanSettings()->SetRowsBatchSize(1); - record.SetTxBody(schemeTx.SerializeAsString()); - } - - return TTestActorRuntime::EEventAction::PROCESS; - } - - case TEvDataShard::EvProposeTransactionResult: { - if (!backupTxId) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - const auto& record = ev->Get<TEvDataShard::TEvProposeTransactionResult>()->Record; - if (record.GetTxId() != *backupTxId) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - tabletId = record.GetOrigin(); - return TTestActorRuntime::EEventAction::PROCESS; - } - - case NWrappers::TEvS3Wrapper::EvCompleteMultipartUploadRequest: - case NWrappers::TEvS3Wrapper::EvAbortMultipartUploadRequest: - delayed = true; - return TTestActorRuntime::EEventAction::DROP; - - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port).WithRejectUploadParts(rejectUploadParts)); - UNIT_ASSERT(s3Mock.Start()); - - TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/Table" - destination_prefix: "" - } - } - )", port)); - const ui64 exportId = txId; - - if (!delayed || !tabletId) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed, &tabletId](IEventHandle&) -> bool { - return delayed && tabletId; - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - + + runtime.SetLogPriority(NKikimrServices::S3_WRAPPER, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); + + TMaybe<ui64> backupTxId; + TMaybe<ui64> tabletId; + bool delayed = false; + + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvDataShard::EvProposeTransaction: { + auto& record = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record; + if (record.GetTxKind() != NKikimrTxDataShard::ETransactionKind::TX_KIND_SCHEME) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + NKikimrTxDataShard::TFlatSchemeTransaction schemeTx; + UNIT_ASSERT(schemeTx.ParseFromString(record.GetTxBody())); + + if (schemeTx.HasBackup()) { + backupTxId = record.GetTxId(); + // hijack + schemeTx.MutableBackup()->MutableScanSettings()->SetRowsBatchSize(1); + record.SetTxBody(schemeTx.SerializeAsString()); + } + + return TTestActorRuntime::EEventAction::PROCESS; + } + + case TEvDataShard::EvProposeTransactionResult: { + if (!backupTxId) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + const auto& record = ev->Get<TEvDataShard::TEvProposeTransactionResult>()->Record; + if (record.GetTxId() != *backupTxId) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + tabletId = record.GetOrigin(); + return TTestActorRuntime::EEventAction::PROCESS; + } + + case NWrappers::TEvS3Wrapper::EvCompleteMultipartUploadRequest: + case NWrappers::TEvS3Wrapper::EvAbortMultipartUploadRequest: + delayed = true; + return TTestActorRuntime::EEventAction::DROP; + + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port).WithRejectUploadParts(rejectUploadParts)); + UNIT_ASSERT(s3Mock.Start()); + + TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "" + } + } + )", port)); + const ui64 exportId = txId; + + if (!delayed || !tabletId) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed, &tabletId](IEventHandle&) -> bool { + return delayed && tabletId; + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + RebootTablet(runtime, *tabletId, runtime.AllocateEdgeActor()); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", expectedStatus); - - TestForgetExport(runtime, ++txId, "/MyRoot", exportId); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - - Y_UNIT_TEST(RebootDuringCompletion) { - RebootDuringFinish(false, Ydb::StatusIds::SUCCESS); - } - - Y_UNIT_TEST(RebootDuringAbortion) { - RebootDuringFinish(true, Ydb::StatusIds::CANCELLED); - } - - Y_UNIT_TEST(ShouldExcludeBackupTableFromStats) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - auto writeRow = [&](ui64 tabletId, const TString& key, const TString& value) { - NKikimrMiniKQL::TResult result; - TString error; - NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( - ( - (let key '( '('key (Utf8 '%s) ) ) ) - (let row '( '('value (Utf8 '%s) ) ) ) - (return (AsList (UpdateRow '__user__Table key row) )) - ) - )", key.c_str(), value.c_str()), result, error); - - UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - }; - - THashSet<ui64> statsCollected; - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvDataShard::EvPeriodicTableStats) { - statsCollected.insert(ev->Get<TEvDataShard::TEvPeriodicTableStats>()->Record.GetDatashardId()); - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - auto waitForStats = [&](ui32 count) { - statsCollected.clear(); - - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&](IEventHandle&) -> bool { - return statsCollected.size() == count; - }); - runtime.DispatchEvents(opts); - - return DescribePath(runtime, "/MyRoot") - .GetPathDescription() - .GetDomainDescription() - .GetDiskSpaceUsage(); - }; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - for (int i = 1; i < 500; ++i) { + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", expectedStatus); + + TestForgetExport(runtime, ++txId, "/MyRoot", exportId); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + + Y_UNIT_TEST(RebootDuringCompletion) { + RebootDuringFinish(false, Ydb::StatusIds::SUCCESS); + } + + Y_UNIT_TEST(RebootDuringAbortion) { + RebootDuringFinish(true, Ydb::StatusIds::CANCELLED); + } + + Y_UNIT_TEST(ShouldExcludeBackupTableFromStats) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + auto writeRow = [&](ui64 tabletId, const TString& key, const TString& value) { + NKikimrMiniKQL::TResult result; + TString error; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( + ( + (let key '( '('key (Utf8 '%s) ) ) ) + (let row '( '('value (Utf8 '%s) ) ) ) + (return (AsList (UpdateRow '__user__Table key row) )) + ) + )", key.c_str(), value.c_str()), result, error); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }; + + THashSet<ui64> statsCollected; + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvDataShard::EvPeriodicTableStats) { + statsCollected.insert(ev->Get<TEvDataShard::TEvPeriodicTableStats>()->Record.GetDatashardId()); + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + auto waitForStats = [&](ui32 count) { + statsCollected.clear(); + + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&](IEventHandle&) -> bool { + return statsCollected.size() == count; + }); + runtime.DispatchEvents(opts); + + return DescribePath(runtime, "/MyRoot") + .GetPathDescription() + .GetDomainDescription() + .GetDiskSpaceUsage(); + }; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + for (int i = 1; i < 500; ++i) { writeRow(TTestTxConfig::FakeHiveTablets, Sprintf("a%i", i), "value"); - } - - // trigger memtable's compaction - TestCopyTable(runtime, ++txId, "/MyRoot", "CopyTable", "/MyRoot/Table"); - env.TestWaitNotification(runtime, txId); - TestDropTable(runtime, ++txId, "/MyRoot", "Table"); - env.TestWaitNotification(runtime, txId); - - const auto expected = waitForStats(1); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/CopyTable" - destination_prefix: "" - } - } - )", port)); - const ui64 exportId = txId; - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::SUCCESS); - const auto afterExport = waitForStats(2); - UNIT_ASSERT_STRINGS_EQUAL(expected.DebugString(), afterExport.DebugString()); - - TestForgetExport(runtime, ++txId, "/MyRoot", exportId); - env.TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - const auto afterForget = waitForStats(1); - UNIT_ASSERT_STRINGS_EQUAL(expected.DebugString(), afterForget.DebugString()); - } + } + + // trigger memtable's compaction + TestCopyTable(runtime, ++txId, "/MyRoot", "CopyTable", "/MyRoot/Table"); + env.TestWaitNotification(runtime, txId); + TestDropTable(runtime, ++txId, "/MyRoot", "Table"); + env.TestWaitNotification(runtime, txId); + + const auto expected = waitForStats(1); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/CopyTable" + destination_prefix: "" + } + } + )", port)); + const ui64 exportId = txId; + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::SUCCESS); + const auto afterExport = waitForStats(2); + UNIT_ASSERT_STRINGS_EQUAL(expected.DebugString(), afterExport.DebugString()); + + TestForgetExport(runtime, ++txId, "/MyRoot", exportId); + env.TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + const auto afterForget = waitForStats(1); + UNIT_ASSERT_STRINGS_EQUAL(expected.DebugString(), afterForget.DebugString()); + } } diff --git a/ydb/core/tx/schemeshard/ut_export/ya.make b/ydb/core/tx/schemeshard/ut_export/ya.make index ca5d16eb35d..6750205274c 100644 --- a/ydb/core/tx/schemeshard/ut_export/ya.make +++ b/ydb/core/tx/schemeshard/ut_export/ya.make @@ -20,21 +20,21 @@ ENDIF() INCLUDE(${ARCADIA_ROOT}/ydb/tests/supp/ubsan_supp.inc) -IF (NOT OS_WINDOWS) - PEERDIR( +IF (NOT OS_WINDOWS) + PEERDIR( library/cpp/getopt - library/cpp/regex/pcre - library/cpp/svnversion + library/cpp/regex/pcre + library/cpp/svnversion ydb/core/testlib ydb/core/tx ydb/core/tx/schemeshard/ut_helpers ydb/core/wrappers/ut_helpers ydb/library/yql/public/udf/service/exception_policy - ) - SRCS( - ut_export.cpp - ) -ENDIF() + ) + SRCS( + ut_export.cpp + ) +ENDIF() YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp index 6c4e367045f..5e1e42d17ce 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3.cpp @@ -5,7 +5,7 @@ #include <util/string/printf.h> using namespace NSchemeShardUT_Private; -using namespace NSchemeShardUT_Private::NExportReboots; +using namespace NSchemeShardUT_Private::NExportReboots; using namespace NKikimr::NWrappers::NTestHelpers; Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp index 89b90c0fc2d..a2d876fd053 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp @@ -1,120 +1,120 @@ -#include "export_reboots_common.h" - +#include "export_reboots_common.h" + #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - -namespace NSchemeShardUT_Private { -namespace NExportReboots { - + +namespace NSchemeShardUT_Private { +namespace NExportReboots { + void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - TSet<ui64> toWait; - for (const auto& table : tables) { - TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); - toWait.insert(t.TxId); - } - t.TestEnv->TestWaitNotification(runtime, toWait); - } - - TestExport(runtime, ++t.TxId, "/MyRoot", request); - const ui64 exportId = t.TxId; - - t.TestEnv->TestWaitNotification(runtime, exportId); - - { - TInactiveZone inactive(activeZone); - - auto response = TestGetExport(runtime, exportId, "/MyRoot", { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::NOT_FOUND - }); - - if (response.GetResponse().GetEntry().GetStatus() == Ydb::StatusIds::NOT_FOUND) { - return; - } - - TestForgetExport(runtime, ++t.TxId, "/MyRoot", exportId); - t.TestEnv->TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - }); -} - + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TSet<ui64> toWait; + for (const auto& table : tables) { + TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); + toWait.insert(t.TxId); + } + t.TestEnv->TestWaitNotification(runtime, toWait); + } + + TestExport(runtime, ++t.TxId, "/MyRoot", request); + const ui64 exportId = t.TxId; + + t.TestEnv->TestWaitNotification(runtime, exportId); + + { + TInactiveZone inactive(activeZone); + + auto response = TestGetExport(runtime, exportId, "/MyRoot", { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::NOT_FOUND + }); + + if (response.GetResponse().GetEntry().GetStatus() == Ydb::StatusIds::NOT_FOUND) { + return; + } + + TestForgetExport(runtime, ++t.TxId, "/MyRoot", exportId); + t.TestEnv->TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + }); +} + void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - TSet<ui64> toWait; - for (const auto& table : tables) { - TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); - toWait.insert(t.TxId); - } - t.TestEnv->TestWaitNotification(runtime, toWait); - } - - TestExport(runtime, ++t.TxId, "/MyRoot", request); - const ui64 exportId = t.TxId; - - t.TestEnv->ReliablePropose(runtime, CancelExportRequest(++t.TxId, "/MyRoot", exportId), { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::NOT_FOUND - }); - t.TestEnv->TestWaitNotification(runtime, exportId); - - { - TInactiveZone inactive(activeZone); - - auto response = TestGetExport(runtime, exportId, "/MyRoot", { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::CANCELLED, - Ydb::StatusIds::NOT_FOUND - }); - - if (response.GetResponse().GetEntry().GetStatus() == Ydb::StatusIds::NOT_FOUND) { - return; - } - - TestForgetExport(runtime, ++t.TxId, "/MyRoot", exportId); - t.TestEnv->TestWaitNotification(runtime, exportId); - - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - }); -} - + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TSet<ui64> toWait; + for (const auto& table : tables) { + TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); + toWait.insert(t.TxId); + } + t.TestEnv->TestWaitNotification(runtime, toWait); + } + + TestExport(runtime, ++t.TxId, "/MyRoot", request); + const ui64 exportId = t.TxId; + + t.TestEnv->ReliablePropose(runtime, CancelExportRequest(++t.TxId, "/MyRoot", exportId), { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::NOT_FOUND + }); + t.TestEnv->TestWaitNotification(runtime, exportId); + + { + TInactiveZone inactive(activeZone); + + auto response = TestGetExport(runtime, exportId, "/MyRoot", { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::CANCELLED, + Ydb::StatusIds::NOT_FOUND + }); + + if (response.GetResponse().GetEntry().GetStatus() == Ydb::StatusIds::NOT_FOUND) { + return; + } + + TestForgetExport(runtime, ++t.TxId, "/MyRoot", exportId); + t.TestEnv->TestWaitNotification(runtime, exportId); + + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + }); +} + void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t) { - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - TSet<ui64> toWait; - for (const auto& table : tables) { - TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); - toWait.insert(t.TxId); - } - t.TestEnv->TestWaitNotification(runtime, toWait); - - TestExport(runtime, ++t.TxId, "/MyRoot", request); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - const ui64 exportId = t.TxId; - - t.TestEnv->ReliablePropose(runtime, ForgetExportRequest(++t.TxId, "/MyRoot", exportId), { - Ydb::StatusIds::SUCCESS, - }); - t.TestEnv->TestWaitNotification(runtime, exportId); - - { - TInactiveZone inactive(activeZone); - TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); - } - }); -} - -} // NExportReboots -} // NSchemeShardUT_Private + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TSet<ui64> toWait; + for (const auto& table : tables) { + TestCreateTable(runtime, ++t.TxId, "/MyRoot", table); + toWait.insert(t.TxId); + } + t.TestEnv->TestWaitNotification(runtime, toWait); + + TestExport(runtime, ++t.TxId, "/MyRoot", request); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + const ui64 exportId = t.TxId; + + t.TestEnv->ReliablePropose(runtime, ForgetExportRequest(++t.TxId, "/MyRoot", exportId), { + Ydb::StatusIds::SUCCESS, + }); + t.TestEnv->TestWaitNotification(runtime, exportId); + + { + TInactiveZone inactive(activeZone); + TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); + } + }); +} + +} // NExportReboots +} // NSchemeShardUT_Private diff --git a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h index a82e62951b7..a7f869c5851 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h +++ b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.h @@ -1,17 +1,17 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NSchemeShardUT_Private { +#pragma once + +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NSchemeShardUT_Private { class TTestWithReboots; -namespace NExportReboots { - +namespace NExportReboots { + void Run(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); void Cancel(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); void Forget(const TVector<TString>& tables, const TString& request, TTestWithReboots& t); - -} // NExportReboots -} // NSchemeShardUT_Private + +} // NExportReboots +} // NSchemeShardUT_Private diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 91b00020c91..6f999b268bb 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -11,87 +11,87 @@ #include <ydb/core/util/pb.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/maybe.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/maybe.h> #include <util/generic/ptr.h> -#include <util/string/split.h> -#include <util/system/env.h> +#include <util/string/split.h> +#include <util/system/env.h> namespace NSchemeShardUT_Private { using namespace NKikimr; - template <typename TEvResponse, typename TEvRequest, typename TStatus> - static ui32 ReliableProposeImpl( - NActors::TTestActorRuntime& runtime, const TActorId& proposer, - TEvRequest* evRequest, const TVector<TStatus>& expectedStatuses) + template <typename TEvResponse, typename TEvRequest, typename TStatus> + static ui32 ReliableProposeImpl( + NActors::TTestActorRuntime& runtime, const TActorId& proposer, + TEvRequest* evRequest, const TVector<TStatus>& expectedStatuses) { TActorId sender = runtime.AllocateEdgeActor(); - ui64 txId = evRequest->Record.GetTxId(); - runtime.Send(new IEventHandle(proposer, sender, evRequest)); + ui64 txId = evRequest->Record.GetTxId(); + runtime.Send(new IEventHandle(proposer, sender, evRequest)); - auto evResponse = runtime.GrabEdgeEvent<TEvResponse>(sender); - UNIT_ASSERT(evResponse); + auto evResponse = runtime.GrabEdgeEvent<TEvResponse>(sender); + UNIT_ASSERT(evResponse); - const auto& record = evResponse->Get()->Record; - UNIT_ASSERT(record.GetTxId() == txId); + const auto& record = evResponse->Get()->Record; + UNIT_ASSERT(record.GetTxId() == txId); - ui32 result = 0; + ui32 result = 0; if constexpr (std::is_same_v<TEvSchemeShard::TEvModifySchemeTransactionResult, TEvResponse>) { - result = record.GetStatus(); - CheckExpected(expectedStatuses, record.GetStatus(), record.GetReason()); + result = record.GetStatus(); + CheckExpected(expectedStatuses, record.GetStatus(), record.GetReason()); } else if constexpr (std::is_same_v<TEvSchemeShard::TEvCancelTxResult, TEvResponse>) { - result = record.GetStatus(); - CheckExpected(expectedStatuses, record.GetStatus(), record.GetResult()); - } else { - result = record.GetResponse().GetStatus(); - CheckExpected(expectedStatuses, record.GetResponse().GetStatus(), "unexpected"); - } + result = record.GetStatus(); + CheckExpected(expectedStatuses, record.GetStatus(), record.GetResult()); + } else { + result = record.GetResponse().GetStatus(); + CheckExpected(expectedStatuses, record.GetResponse().GetStatus(), "unexpected"); + } - return result; + return result; } - ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( + ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( NActors::TTestActorRuntime& runtime, TEvSchemeShard::TEvModifySchemeTransaction* evTx, const TVector<TEvSchemeShard::EStatus>& expectedResults) - { + { return ReliableProposeImpl<TEvSchemeShard::TEvModifySchemeTransactionResult>( - runtime, TxReliablePropose, evTx, expectedResults); - } - - ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( + runtime, TxReliablePropose, evTx, expectedResults); + } + + ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( NActors::TTestActorRuntime& runtime, TEvSchemeShard::TEvCancelTx* evTx, const TVector<TEvSchemeShard::EStatus>& expectedResults) - { + { return ReliableProposeImpl<TEvSchemeShard::TEvCancelTxResult>( - runtime, TxReliablePropose, evTx, expectedResults); - } - - ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( - NActors::TTestActorRuntime& runtime, TEvExport::TEvCancelExportRequest* ev, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) - { - return ReliableProposeImpl<TEvExport::TEvCancelExportResponse>( - runtime, TxReliablePropose, ev, expectedStatuses); - } - - ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( - NActors::TTestActorRuntime& runtime, TEvExport::TEvForgetExportRequest* ev, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) - { - return ReliableProposeImpl<TEvExport::TEvForgetExportResponse>( - runtime, TxReliablePropose, ev, expectedStatuses); - } - - ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( - NActors::TTestActorRuntime& runtime, TEvImport::TEvCancelImportRequest* ev, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) - { - return ReliableProposeImpl<TEvImport::TEvCancelImportResponse>( - runtime, TxReliablePropose, ev, expectedStatuses); - } - + runtime, TxReliablePropose, evTx, expectedResults); + } + + ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( + NActors::TTestActorRuntime& runtime, TEvExport::TEvCancelExportRequest* ev, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) + { + return ReliableProposeImpl<TEvExport::TEvCancelExportResponse>( + runtime, TxReliablePropose, ev, expectedStatuses); + } + + ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( + NActors::TTestActorRuntime& runtime, TEvExport::TEvForgetExportRequest* ev, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) + { + return ReliableProposeImpl<TEvExport::TEvForgetExportResponse>( + runtime, TxReliablePropose, ev, expectedStatuses); + } + + ui32 NSchemeShardUT_Private::TTestEnv::ReliablePropose( + NActors::TTestActorRuntime& runtime, TEvImport::TEvCancelImportRequest* ev, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) + { + return ReliableProposeImpl<TEvImport::TEvCancelImportResponse>( + runtime, TxReliablePropose, ev, expectedStatuses); + } + NKikimrSchemeOp::TAlterUserAttributes AlterUserAttrs(const TVector<std::pair<TString, TString>>& add, const TVector<TString>& drop) { NKikimrSchemeOp::TAlterUserAttributes result; for (const auto& item: add) { @@ -160,9 +160,9 @@ namespace NSchemeShardUT_Private { auto op = transaction->MutableModifyACL(); op->SetName(name); - if (diffAcl) { - op->SetDiffACL(diffAcl); - } + if (diffAcl) { + op->SetDiffACL(diffAcl); + } if (newOwner) { op->SetNewOwner(newOwner); } @@ -197,7 +197,7 @@ namespace NSchemeShardUT_Private { NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, const NKikimrSchemeOp::TDescribeOptions& opts) { TActorId sender = runtime.AllocateEdgeActor(); auto evLs = new TEvSchemeShard::TEvDescribeScheme(path); - evLs->Record.MutableOptions()->CopyFrom(opts); + evLs->Record.MutableOptions()->CopyFrom(opts); ForwardToTablet(runtime, schemeShard, sender, evLs); TAutoPtr<IEventHandle> handle; auto event = runtime.GrabEdgeEvent<TEvSchemeShard::TEvDescribeSchemeResult>(handle); @@ -220,32 +220,32 @@ namespace NSchemeShardUT_Private { NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, const NKikimrSchemeOp::TDescribeOptions& opts) { return DescribePath(runtime, TTestTxConfig::SchemeShard, path, opts); - } - + } + NKikimrScheme::TEvDescribeSchemeResult DescribePathId(TTestActorRuntime& runtime, ui64 pathId, const NKikimrSchemeOp::TDescribeOptions& opts = { }) { return DescribePathId(runtime, TTestTxConfig::SchemeShard, pathId, opts); } - NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning, bool returnBoundaries) { - return DescribePath(runtime, schemeShard, path, returnPartitioning, returnBoundaries, true); - } - - NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning, bool returnBoundaries, bool showPrivate, bool returnBackups) { + NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning, bool returnBoundaries) { + return DescribePath(runtime, schemeShard, path, returnPartitioning, returnBoundaries, true); + } + + NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning, bool returnBoundaries, bool showPrivate, bool returnBackups) { NKikimrSchemeOp::TDescribeOptions opts; - opts.SetReturnPartitioningInfo(returnPartitioning); - opts.SetReturnPartitionConfig(returnPartitioning); - opts.SetBackupInfo(returnBackups); - opts.SetReturnBoundaries(returnBoundaries); - opts.SetShowPrivateTable(showPrivate); - - return DescribePath(runtime, schemeShard, path, opts); - } - - NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning, bool returnBoundaries) { + opts.SetReturnPartitioningInfo(returnPartitioning); + opts.SetReturnPartitionConfig(returnPartitioning); + opts.SetBackupInfo(returnBackups); + opts.SetReturnBoundaries(returnBoundaries); + opts.SetShowPrivateTable(showPrivate); + + return DescribePath(runtime, schemeShard, path, opts); + } + + NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning, bool returnBoundaries) { return DescribePath(runtime, TTestTxConfig::SchemeShard, path, returnPartitioning, returnBoundaries, true); } - NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning, bool returnBoundaries, bool showPrivate, bool returnBackups) { + NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning, bool returnBoundaries, bool showPrivate, bool returnBackups) { return DescribePath(runtime, TTestTxConfig::SchemeShard, path, returnPartitioning, returnBoundaries, showPrivate, returnBackups); } @@ -427,384 +427,384 @@ namespace NSchemeShardUT_Private { TestUnlock(runtime, TTestTxConfig::SchemeShard, txId, lockId, parentPath, name, expectedResults); } - template <typename T> - using TModifySchemeFunc = T*(NKikimrSchemeOp::TModifyScheme::*)(); - - // Generic - template <typename T> - auto CreateTransaction(const TString& parentPath, const TString& scheme, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<T> func) - { - NKikimrSchemeOp::TModifyScheme tx; - - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - SetApplyIf(tx, applyIf); - - const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); - UNIT_ASSERT_C(ok, "protobuf parsing failed"); - - return tx; - } - - // SplitMerge - template <> - auto CreateTransaction(const TString& tablePath, const TString& scheme, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TSplitMergeTablePartitions> func) - { - NKikimrSchemeOp::TModifyScheme tx; - - tx.SetOperationType(type); - SetApplyIf(tx, applyIf); - - const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); - UNIT_ASSERT_C(ok, "protobuf parsing failed"); - tx.MutableSplitMergeTablePartitions()->SetTablePath(tablePath); - - return tx; - } - - // Drop - template <> - auto CreateTransaction(const TString& parentPath, const TString& name, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TDrop> func) - { - NKikimrSchemeOp::TModifyScheme tx; - - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - SetApplyIf(tx, applyIf); - - std::apply(func, std::tie(tx))->SetName(name); - - return tx; - } - - // Backup - template <> - auto CreateTransaction(const TString& parentPath, const TString& tableName, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TBackupTask> func) + template <typename T> + using TModifySchemeFunc = T*(NKikimrSchemeOp::TModifyScheme::*)(); + + // Generic + template <typename T> + auto CreateTransaction(const TString& parentPath, const TString& scheme, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<T> func) { - NKikimrSchemeOp::TModifyScheme tx; + NKikimrSchemeOp::TModifyScheme tx; - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - SetApplyIf(tx, applyIf); + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + SetApplyIf(tx, applyIf); - auto task = std::apply(func, std::tie(tx)); + const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); + UNIT_ASSERT_C(ok, "protobuf parsing failed"); + + return tx; + } - task->SetTableName(tableName); - auto& settings = *task->MutableYTSettings(); + // SplitMerge + template <> + auto CreateTransaction(const TString& tablePath, const TString& scheme, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TSplitMergeTablePartitions> func) + { + NKikimrSchemeOp::TModifyScheme tx; - TString ytHost; - TMaybe<ui16> ytPort; - Split(GetEnv("YT_PROXY"), ':', ytHost, ytPort); + tx.SetOperationType(type); + SetApplyIf(tx, applyIf); - settings.SetHost(ytHost); - settings.SetPort(ytPort.GetOrElse(80)); - settings.SetTablePattern("<append=true>//tmp/table"); + const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); + UNIT_ASSERT_C(ok, "protobuf parsing failed"); + tx.MutableSplitMergeTablePartitions()->SetTablePath(tablePath); - return tx; + return tx; } - // Generic with attrs - template <typename T> - auto CreateTransaction(const TString& parentPath, const TString& scheme, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<T> func) + // Drop + template <> + auto CreateTransaction(const TString& parentPath, const TString& name, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TDrop> func) { - NKikimrSchemeOp::TModifyScheme tx; + NKikimrSchemeOp::TModifyScheme tx; - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - if (userAttrs.UserAttributesSize()) { - tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); - } - SetApplyIf(tx, applyIf); + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + SetApplyIf(tx, applyIf); - const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); - UNIT_ASSERT_C(ok, "protobuf parsing failed"); + std::apply(func, std::tie(tx))->SetName(name); - return tx; + return tx; } - // MkDir - template <> - auto CreateTransaction(const TString& parentPath, const TString& name, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TMkDir> func) + // Backup + template <> + auto CreateTransaction(const TString& parentPath, const TString& tableName, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TBackupTask> func) { - NKikimrSchemeOp::TModifyScheme tx; + NKikimrSchemeOp::TModifyScheme tx; + + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + SetApplyIf(tx, applyIf); + + auto task = std::apply(func, std::tie(tx)); - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - if (userAttrs.UserAttributesSize()) { - tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); - } - SetApplyIf(tx, applyIf); + task->SetTableName(tableName); + auto& settings = *task->MutableYTSettings(); - std::apply(func, std::tie(tx))->SetName(name); + TString ytHost; + TMaybe<ui16> ytPort; + Split(GetEnv("YT_PROXY"), ':', ytHost, ytPort); - return tx; + settings.SetHost(ytHost); + settings.SetPort(ytPort.GetOrElse(80)); + settings.SetTablePattern("<append=true>//tmp/table"); + + return tx; } - // AlterUserAttrs - template <> - auto CreateTransaction(const TString& parentPath, const TString& name, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, - NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TAlterUserAttributes> func) + // Generic with attrs + template <typename T> + auto CreateTransaction(const TString& parentPath, const TString& scheme, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<T> func) { - NKikimrSchemeOp::TModifyScheme tx; + NKikimrSchemeOp::TModifyScheme tx; - tx.SetOperationType(type); - tx.SetWorkingDir(parentPath); - tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); - SetApplyIf(tx, applyIf); + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + if (userAttrs.UserAttributesSize()) { + tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); + } + SetApplyIf(tx, applyIf); - std::apply(func, std::tie(tx))->SetPathName(name); + const bool ok = google::protobuf::TextFormat::ParseFromString(scheme, std::apply(func, std::tie(tx))); + UNIT_ASSERT_C(ok, "protobuf parsing failed"); - return tx; + return tx; } - // Drop by pathId - auto CreateTransaction(ui64 pathId, const TApplyIf& applyIf, NKikimrSchemeOp::EOperationType type) { - NKikimrSchemeOp::TModifyScheme tx; + // MkDir + template <> + auto CreateTransaction(const TString& parentPath, const TString& name, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TMkDir> func) + { + NKikimrSchemeOp::TModifyScheme tx; + + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + if (userAttrs.UserAttributesSize()) { + tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); + } + SetApplyIf(tx, applyIf); - tx.SetOperationType(type); - tx.MutableDrop()->SetId(pathId); - SetApplyIf(tx, applyIf); + std::apply(func, std::tie(tx))->SetName(name); - return tx; + return tx; } - TEvTx* CreateRequest(ui64 schemeShardId, ui64 txId, NKikimrSchemeOp::TModifyScheme&& tx) { - auto ev = new TEvTx(txId, schemeShardId); - *ev->Record.AddTransaction() = std::move(tx); + // AlterUserAttrs + template <> + auto CreateTransaction(const TString& parentPath, const TString& name, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf, + NKikimrSchemeOp::EOperationType type, TModifySchemeFunc<NKikimrSchemeOp::TAlterUserAttributes> func) + { + NKikimrSchemeOp::TModifyScheme tx; + + tx.SetOperationType(type); + tx.SetWorkingDir(parentPath); + tx.MutableAlterUserAttributes()->CopyFrom(userAttrs); + SetApplyIf(tx, applyIf); - return ev; + std::apply(func, std::tie(tx))->SetPathName(name); + + return tx; + } + + // Drop by pathId + auto CreateTransaction(ui64 pathId, const TApplyIf& applyIf, NKikimrSchemeOp::EOperationType type) { + NKikimrSchemeOp::TModifyScheme tx; + + tx.SetOperationType(type); + tx.MutableDrop()->SetId(pathId); + SetApplyIf(tx, applyIf); + + return tx; } - #define GENERIC_HELPERS(name, op, func) \ - TEvTx* name##Request(ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ - return CreateRequest(schemeShardId, txId, CreateTransaction(parentPath, scheme, applyIf, op, func)); \ - } \ - \ - TEvTx* name##Request(ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ + TEvTx* CreateRequest(ui64 schemeShardId, ui64 txId, NKikimrSchemeOp::TModifyScheme&& tx) { + auto ev = new TEvTx(txId, schemeShardId); + *ev->Record.AddTransaction() = std::move(tx); + + return ev; + } + + #define GENERIC_HELPERS(name, op, func) \ + TEvTx* name##Request(ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ + return CreateRequest(schemeShardId, txId, CreateTransaction(parentPath, scheme, applyIf, op, func)); \ + } \ + \ + TEvTx* name##Request(ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ return name##Request(TTestTxConfig::SchemeShard, txId, parentPath, scheme, applyIf); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ ForwardToTablet(runtime, schemeShardId, runtime.AllocateEdgeActor(), \ - name##Request(schemeShardId, txId, parentPath, scheme, applyIf)); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ + name##Request(schemeShardId, txId, parentPath, scheme, applyIf)); \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, const TApplyIf& applyIf) { \ Async##name(runtime, TTestTxConfig::SchemeShard, txId, parentPath, scheme, applyIf); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ - { \ - Async##name(runtime, schemeShardId, txId, parentPath, scheme, applyIf); \ - return TestModificationResults(runtime, txId, expectedResults); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ - { \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ + { \ + Async##name(runtime, schemeShardId, txId, parentPath, scheme, applyIf); \ + return TestModificationResults(runtime, txId, expectedResults); \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ + { \ return Test##name(runtime, TTestTxConfig::SchemeShard, txId, parentPath, scheme, expectedResults, applyIf); \ - } - - #define GENERIC_WITH_ATTRS_HELPERS(name, op, func) \ - TEvTx* name##Request(ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ - return CreateRequest(schemeShardId, txId, CreateTransaction(parentPath, scheme, userAttrs, applyIf, op, func)); \ - } \ - \ - TEvTx* name##Request(ui64 txId, const TString& parentPath, const TString& scheme, \ - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ + } + + #define GENERIC_WITH_ATTRS_HELPERS(name, op, func) \ + TEvTx* name##Request(ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ + return CreateRequest(schemeShardId, txId, CreateTransaction(parentPath, scheme, userAttrs, applyIf, op, func)); \ + } \ + \ + TEvTx* name##Request(ui64 txId, const TString& parentPath, const TString& scheme, \ + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ return name##Request(TTestTxConfig::SchemeShard, txId, parentPath, scheme, userAttrs, applyIf); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ - TEvTx* req = name##Request(schemeShardId, txId, parentPath, scheme, userAttrs, applyIf); \ - if (req->Record.GetTransaction(0).GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain) { \ - for (const auto& [kind, pool] : runtime.GetAppData().DomainsInfo->GetDomain(0).StoragePoolTypes) { \ - auto* pbPool = req->Record.MutableTransaction(0)->MutableSubDomain()->AddStoragePools(); \ - pbPool->SetKind(kind); \ - pbPool->SetName(pool.GetName()); \ - } \ - } \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ + TEvTx* req = name##Request(schemeShardId, txId, parentPath, scheme, userAttrs, applyIf); \ + if (req->Record.GetTransaction(0).GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain) { \ + for (const auto& [kind, pool] : runtime.GetAppData().DomainsInfo->GetDomain(0).StoragePoolTypes) { \ + auto* pbPool = req->Record.MutableTransaction(0)->MutableSubDomain()->AddStoragePools(); \ + pbPool->SetKind(kind); \ + pbPool->SetName(pool.GetName()); \ + } \ + } \ ForwardToTablet(runtime, schemeShardId, runtime.AllocateEdgeActor(), req); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ Async##name(runtime, TTestTxConfig::SchemeShard, txId, parentPath, scheme, userAttrs, applyIf); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ - Async##name(runtime, schemeShardId, txId, parentPath, scheme, userAttrs, applyIf); \ - return TestModificationResults(runtime, txId, expectedResults); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ - { \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString& parentPath, const TString& scheme, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ + Async##name(runtime, schemeShardId, txId, parentPath, scheme, userAttrs, applyIf); \ + return TestModificationResults(runtime, txId, expectedResults); \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs, const TApplyIf& applyIf) \ + { \ return Test##name(runtime, TTestTxConfig::SchemeShard, txId, parentPath, scheme, expectedResults, userAttrs, applyIf); \ - } - - #define DROP_BY_PATH_ID_HELPERS(name, op) \ - TEvTx* name##Request(ui64 schemeShardId, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ - return CreateRequest(schemeShardId, txId, CreateTransaction(pathId, applyIf, op)); \ - } \ - \ - TEvTx* name##Request(ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ + } + + #define DROP_BY_PATH_ID_HELPERS(name, op) \ + TEvTx* name##Request(ui64 schemeShardId, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ + return CreateRequest(schemeShardId, txId, CreateTransaction(pathId, applyIf, op)); \ + } \ + \ + TEvTx* name##Request(ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ return name##Request(TTestTxConfig::SchemeShard, txId, pathId, applyIf); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ ForwardToTablet(runtime, schemeShardId, runtime.AllocateEdgeActor(), \ - name##Request(schemeShardId, txId, pathId, applyIf)); \ - } \ - \ - void Async##name(TTestActorRuntime& runtime, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ + name##Request(schemeShardId, txId, pathId, applyIf)); \ + } \ + \ + void Async##name(TTestActorRuntime& runtime, ui64 txId, ui64 pathId, const TApplyIf& applyIf) { \ Async##name(runtime, TTestTxConfig::SchemeShard, txId, pathId, applyIf); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, ui64 pathId, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ - { \ - Async##name(runtime, schemeShardId, txId, pathId, applyIf); \ - return TestModificationResults(runtime, txId, expectedResults); \ - } \ - \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, ui64 pathId, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ - { \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, ui64 pathId, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ + { \ + Async##name(runtime, schemeShardId, txId, pathId, applyIf); \ + return TestModificationResults(runtime, txId, expectedResults); \ + } \ + \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, ui64 pathId, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults, const TApplyIf& applyIf) \ + { \ return Test##name(runtime, TTestTxConfig::SchemeShard, txId, pathId, expectedResults, applyIf); \ - } - - // subdomain - GENERIC_WITH_ATTRS_HELPERS(CreateSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) - GENERIC_HELPERS(AlterSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpAlterSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) - GENERIC_HELPERS(DropSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpDropSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - GENERIC_HELPERS(ForceDropSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - - // ext subdomain - GENERIC_WITH_ATTRS_HELPERS(CreateExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpCreateExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) - GENERIC_WITH_ATTRS_HELPERS(AlterExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpAlterExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) - GENERIC_HELPERS(ForceDropExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - - // dir - GENERIC_WITH_ATTRS_HELPERS(MkDir, NKikimrSchemeOp::EOperationType::ESchemeOpMkDir, &NKikimrSchemeOp::TModifyScheme::MutableMkDir) - GENERIC_HELPERS(RmDir, NKikimrSchemeOp::EOperationType::ESchemeOpRmDir, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(ForceDropUnsafe, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropUnsafe) - - // user attrs - GENERIC_WITH_ATTRS_HELPERS(UserAttrs, NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes, &NKikimrSchemeOp::TModifyScheme::MutableAlterUserAttributes) - - // table - GENERIC_WITH_ATTRS_HELPERS(CreateTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateTable) - GENERIC_HELPERS(CreateIndexedTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateIndexedTable) - GENERIC_HELPERS(ConsistentCopyTables, NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables, &NKikimrSchemeOp::TModifyScheme::MutableCreateConsistentCopyTables) - GENERIC_HELPERS(AlterTable, NKikimrSchemeOp::EOperationType::ESchemeOpAlterTable, &NKikimrSchemeOp::TModifyScheme::MutableAlterTable) - GENERIC_HELPERS(SplitTable, NKikimrSchemeOp::EOperationType::ESchemeOpSplitMergeTablePartitions, &NKikimrSchemeOp::TModifyScheme::MutableSplitMergeTablePartitions) - GENERIC_HELPERS(DropTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropTable, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropTable) - GENERIC_HELPERS(DropTableIndex, NKikimrSchemeOp::EOperationType::ESchemeOpDropIndex, &NKikimrSchemeOp::TModifyScheme::MutableDropIndex) - - // backup & restore - GENERIC_HELPERS(BackupTable, NKikimrSchemeOp::EOperationType::ESchemeOpBackup, &NKikimrSchemeOp::TModifyScheme::MutableBackup) - GENERIC_HELPERS(Restore, NKikimrSchemeOp::EOperationType::ESchemeOpRestore, &NKikimrSchemeOp::TModifyScheme::MutableRestore) - - // cdc stream - GENERIC_HELPERS(CreateCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableCreateCdcStream) - GENERIC_HELPERS(AlterCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableAlterCdcStream) - GENERIC_HELPERS(DropCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableDropCdcStream) - - // olap store + } + + // subdomain + GENERIC_WITH_ATTRS_HELPERS(CreateSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) + GENERIC_HELPERS(AlterSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpAlterSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) + GENERIC_HELPERS(DropSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpDropSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + GENERIC_HELPERS(ForceDropSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + + // ext subdomain + GENERIC_WITH_ATTRS_HELPERS(CreateExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpCreateExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) + GENERIC_WITH_ATTRS_HELPERS(AlterExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpAlterExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableSubDomain) + GENERIC_HELPERS(ForceDropExtSubDomain, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropExtSubDomain, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + + // dir + GENERIC_WITH_ATTRS_HELPERS(MkDir, NKikimrSchemeOp::EOperationType::ESchemeOpMkDir, &NKikimrSchemeOp::TModifyScheme::MutableMkDir) + GENERIC_HELPERS(RmDir, NKikimrSchemeOp::EOperationType::ESchemeOpRmDir, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(ForceDropUnsafe, NKikimrSchemeOp::EOperationType::ESchemeOpForceDropUnsafe) + + // user attrs + GENERIC_WITH_ATTRS_HELPERS(UserAttrs, NKikimrSchemeOp::EOperationType::ESchemeOpAlterUserAttributes, &NKikimrSchemeOp::TModifyScheme::MutableAlterUserAttributes) + + // table + GENERIC_WITH_ATTRS_HELPERS(CreateTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateTable) + GENERIC_HELPERS(CreateIndexedTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateIndexedTable) + GENERIC_HELPERS(ConsistentCopyTables, NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables, &NKikimrSchemeOp::TModifyScheme::MutableCreateConsistentCopyTables) + GENERIC_HELPERS(AlterTable, NKikimrSchemeOp::EOperationType::ESchemeOpAlterTable, &NKikimrSchemeOp::TModifyScheme::MutableAlterTable) + GENERIC_HELPERS(SplitTable, NKikimrSchemeOp::EOperationType::ESchemeOpSplitMergeTablePartitions, &NKikimrSchemeOp::TModifyScheme::MutableSplitMergeTablePartitions) + GENERIC_HELPERS(DropTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropTable, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropTable) + GENERIC_HELPERS(DropTableIndex, NKikimrSchemeOp::EOperationType::ESchemeOpDropIndex, &NKikimrSchemeOp::TModifyScheme::MutableDropIndex) + + // backup & restore + GENERIC_HELPERS(BackupTable, NKikimrSchemeOp::EOperationType::ESchemeOpBackup, &NKikimrSchemeOp::TModifyScheme::MutableBackup) + GENERIC_HELPERS(Restore, NKikimrSchemeOp::EOperationType::ESchemeOpRestore, &NKikimrSchemeOp::TModifyScheme::MutableRestore) + + // cdc stream + GENERIC_HELPERS(CreateCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableCreateCdcStream) + GENERIC_HELPERS(AlterCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpAlterCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableAlterCdcStream) + GENERIC_HELPERS(DropCdcStream, NKikimrSchemeOp::EOperationType::ESchemeOpDropCdcStream, &NKikimrSchemeOp::TModifyScheme::MutableDropCdcStream) + + // olap store GENERIC_HELPERS(CreateOlapStore, NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnStore, &NKikimrSchemeOp::TModifyScheme::MutableCreateColumnStore) GENERIC_HELPERS(AlterOlapStore, NKikimrSchemeOp::EOperationType::ESchemeOpAlterColumnStore, &NKikimrSchemeOp::TModifyScheme::MutableAlterColumnStore) GENERIC_HELPERS(DropOlapStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnStore, &NKikimrSchemeOp::TModifyScheme::MutableDrop) DROP_BY_PATH_ID_HELPERS(DropOlapStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnStore) - // olap table + // olap table GENERIC_HELPERS(CreateOlapTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateColumnTable) GENERIC_HELPERS(AlterOlapTable, NKikimrSchemeOp::EOperationType::ESchemeOpAlterColumnTable, &NKikimrSchemeOp::TModifyScheme::MutableAlterColumnTable) GENERIC_HELPERS(DropOlapTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnTable, &NKikimrSchemeOp::TModifyScheme::MutableDrop) DROP_BY_PATH_ID_HELPERS(DropOlapTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropColumnTable) - // sequence - GENERIC_HELPERS(CreateSequence, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence, &NKikimrSchemeOp::TModifyScheme::MutableSequence) - GENERIC_HELPERS(DropSequence, NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropSequence, NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence) - - // replication - GENERIC_HELPERS(CreateReplication, NKikimrSchemeOp::EOperationType::ESchemeOpCreateReplication, &NKikimrSchemeOp::TModifyScheme::MutableReplication) - GENERIC_HELPERS(DropReplication, NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropReplication, NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication) - - // pq - GENERIC_HELPERS(CreatePQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableCreatePersQueueGroup) - GENERIC_HELPERS(AlterPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableAlterPersQueueGroup) - GENERIC_HELPERS(DropPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup) - - // rtmr - GENERIC_HELPERS(CreateRtmrVolume, NKikimrSchemeOp::EOperationType::ESchemeOpCreateRtmrVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateRtmrVolume) - - // solomon - GENERIC_HELPERS(CreateSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateSolomonVolume) - GENERIC_HELPERS(AlterSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpAlterSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableAlterSolomonVolume) - GENERIC_HELPERS(DropSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpDropSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpDropSolomonVolume) - - // kesus - GENERIC_HELPERS(CreateKesus, NKikimrSchemeOp::EOperationType::ESchemeOpCreateKesus, &NKikimrSchemeOp::TModifyScheme::MutableKesus) - GENERIC_HELPERS(AlterKesus, NKikimrSchemeOp::EOperationType::ESchemeOpAlterKesus, &NKikimrSchemeOp::TModifyScheme::MutableKesus) - GENERIC_HELPERS(DropKesus, NKikimrSchemeOp::EOperationType::ESchemeOpDropKesus, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropKesus, NKikimrSchemeOp::EOperationType::ESchemeOpDropKesus) - - // filestore - GENERIC_HELPERS(CreateFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpCreateFileStore, &NKikimrSchemeOp::TModifyScheme::MutableCreateFileStore) - GENERIC_HELPERS(AlterFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpAlterFileStore, &NKikimrSchemeOp::TModifyScheme::MutableAlterFileStore) - GENERIC_HELPERS(DropFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropFileStore, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropFileStore) - - // nbs - GENERIC_HELPERS(CreateBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpCreateBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateBlockStoreVolume) - GENERIC_HELPERS(AlterBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpAlterBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableAlterBlockStoreVolume) - GENERIC_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop) - DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume) - - #undef DROP_BY_PATH_ID_HELPERS - #undef GENERIC_WITH_ATTRS_HELPERS - #undef GENERIC_HELPERS - - ui64 TestCreateSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) + // sequence + GENERIC_HELPERS(CreateSequence, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence, &NKikimrSchemeOp::TModifyScheme::MutableSequence) + GENERIC_HELPERS(DropSequence, NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropSequence, NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence) + + // replication + GENERIC_HELPERS(CreateReplication, NKikimrSchemeOp::EOperationType::ESchemeOpCreateReplication, &NKikimrSchemeOp::TModifyScheme::MutableReplication) + GENERIC_HELPERS(DropReplication, NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropReplication, NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication) + + // pq + GENERIC_HELPERS(CreatePQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpCreatePersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableCreatePersQueueGroup) + GENERIC_HELPERS(AlterPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableAlterPersQueueGroup) + GENERIC_HELPERS(DropPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropPQGroup, NKikimrSchemeOp::EOperationType::ESchemeOpDropPersQueueGroup) + + // rtmr + GENERIC_HELPERS(CreateRtmrVolume, NKikimrSchemeOp::EOperationType::ESchemeOpCreateRtmrVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateRtmrVolume) + + // solomon + GENERIC_HELPERS(CreateSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateSolomonVolume) + GENERIC_HELPERS(AlterSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpAlterSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableAlterSolomonVolume) + GENERIC_HELPERS(DropSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpDropSolomonVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropSolomon, NKikimrSchemeOp::EOperationType::ESchemeOpDropSolomonVolume) + + // kesus + GENERIC_HELPERS(CreateKesus, NKikimrSchemeOp::EOperationType::ESchemeOpCreateKesus, &NKikimrSchemeOp::TModifyScheme::MutableKesus) + GENERIC_HELPERS(AlterKesus, NKikimrSchemeOp::EOperationType::ESchemeOpAlterKesus, &NKikimrSchemeOp::TModifyScheme::MutableKesus) + GENERIC_HELPERS(DropKesus, NKikimrSchemeOp::EOperationType::ESchemeOpDropKesus, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropKesus, NKikimrSchemeOp::EOperationType::ESchemeOpDropKesus) + + // filestore + GENERIC_HELPERS(CreateFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpCreateFileStore, &NKikimrSchemeOp::TModifyScheme::MutableCreateFileStore) + GENERIC_HELPERS(AlterFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpAlterFileStore, &NKikimrSchemeOp::TModifyScheme::MutableAlterFileStore) + GENERIC_HELPERS(DropFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropFileStore, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropFileStore, NKikimrSchemeOp::EOperationType::ESchemeOpDropFileStore) + + // nbs + GENERIC_HELPERS(CreateBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpCreateBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableCreateBlockStoreVolume) + GENERIC_HELPERS(AlterBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpAlterBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableAlterBlockStoreVolume) + GENERIC_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume) + + #undef DROP_BY_PATH_ID_HELPERS + #undef GENERIC_WITH_ATTRS_HELPERS + #undef GENERIC_HELPERS + + ui64 TestCreateSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) { - return TestCreateSubDomain(runtime, txId, parentPath, scheme, {NKikimrScheme::StatusAccepted}, userAttrs); + return TestCreateSubDomain(runtime, txId, parentPath, scheme, {NKikimrScheme::StatusAccepted}, userAttrs); } - ui64 TestCreateExtSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) + ui64 TestCreateExtSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) { - return TestCreateExtSubDomain(runtime, txId, parentPath, scheme, {NKikimrScheme::StatusAccepted}, userAttrs); + return TestCreateExtSubDomain(runtime, txId, parentPath, scheme, {NKikimrScheme::StatusAccepted}, userAttrs); } - ui64 TestUserAttrs(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) + ui64 TestUserAttrs(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs) { - return TestUserAttrs(runtime, txId, parentPath, name, {NKikimrScheme::StatusAccepted}, userAttrs); + return TestUserAttrs(runtime, txId, parentPath, name, {NKikimrScheme::StatusAccepted}, userAttrs); } void AsyncAssignBlockStoreVolume(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, @@ -833,240 +833,240 @@ namespace NSchemeShardUT_Private { TEvSchemeShard::TEvCancelTx *CancelTxRequest(ui64 txId, ui64 targetTxId) { auto evTx = new TEvSchemeShard::TEvCancelTx(); evTx->Record.SetTxId(txId); - evTx->Record.SetTargetTxId(targetTxId); + evTx->Record.SetTargetTxId(targetTxId); return evTx; } - void AsyncCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId) { + void AsyncCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId) { TActorId sender = runtime.AllocateEdgeActor(); ForwardToTablet(runtime, TTestTxConfig::SchemeShard, sender, CancelTxRequest(txId, targetTxId)); } - void TestCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId, + void TestCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId, const TVector<TEvSchemeShard::EStatus>& expectedResults) { - AsyncCancelTxTable(runtime, txId, targetTxId); + AsyncCancelTxTable(runtime, txId, targetTxId); TAutoPtr<IEventHandle> handle; TEvSchemeShard::TEvCancelTxResult* event; do { event = runtime.GrabEdgeEvent<TEvSchemeShard::TEvCancelTxResult>(handle); UNIT_ASSERT(event); - Cerr << "TEvCancelTxResult for TargetTxId: " << event->Record.GetTargetTxId() << ", wait until TargetTxId: " << targetTxId << "\n"; - } while(event->Record.GetTargetTxId() < targetTxId); - UNIT_ASSERT_VALUES_EQUAL(event->Record.GetTargetTxId(), targetTxId); + Cerr << "TEvCancelTxResult for TargetTxId: " << event->Record.GetTargetTxId() << ", wait until TargetTxId: " << targetTxId << "\n"; + } while(event->Record.GetTargetTxId() < targetTxId); + UNIT_ASSERT_VALUES_EQUAL(event->Record.GetTargetTxId(), targetTxId); CheckExpected(expectedResults, event->Record.GetStatus(), event->Record.GetResult()); } - void AsyncExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { - NKikimrExport::TCreateExportRequest request; - UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); - - if (request.HasExportToYtSettings()) { - TString host; - TMaybe<ui16> port; - Split(GetEnv("YT_PROXY"), ':', host, port); - - auto& settings = *request.MutableExportToYtSettings(); - settings.set_host(host); - settings.set_port(port.GetOrElse(80)); - } - - auto ev = MakeHolder<TEvExport::TEvCreateExportRequest>(id, dbName, request); - if (userSID) { - ev->Record.SetUserSID(userSID); - } - + void AsyncExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { + NKikimrExport::TCreateExportRequest request; + UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); + + if (request.HasExportToYtSettings()) { + TString host; + TMaybe<ui16> port; + Split(GetEnv("YT_PROXY"), ':', host, port); + + auto& settings = *request.MutableExportToYtSettings(); + settings.set_host(host); + settings.set_port(port.GetOrElse(80)); + } + + auto ev = MakeHolder<TEvExport::TEvCreateExportRequest>(id, dbName, request); + if (userSID) { + ev->Record.SetUserSID(userSID); + } + ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), ev.Release()); - } - - void AsyncExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { + } + + void AsyncExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { AsyncExport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID); - } - - void TestExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, - Ydb::StatusIds::StatusCode expectedStatus) { - AsyncExport(runtime, schemeshardId, id, dbName, requestStr, userSID); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvExport::TEvCreateExportResponse>(handle); - UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); - } - - void TestExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + void TestExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, + Ydb::StatusIds::StatusCode expectedStatus) { + AsyncExport(runtime, schemeshardId, id, dbName, requestStr, userSID); + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvExport::TEvCreateExportResponse>(handle); + UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); + } + + void TestExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, + Ydb::StatusIds::StatusCode expectedStatus) { TestExport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID, expectedStatus); - } - - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { + } + + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), new TEvExport::TEvGetExportRequest(dbName, id)); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvExport::TEvGetExportResponse>(handle); - const auto result = ev->Record.GetResponse().GetEntry().GetStatus(); - - bool found = false; - for (const auto status : expectedStatuses) { - if (result == status) { - found = true; - break; - } - } - - if (!found) { - UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result)); - } - - return ev->Record; - } - - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvExport::TEvGetExportResponse>(handle); + const auto result = ev->Record.GetResponse().GetEntry().GetStatus(); + + bool found = false; + for (const auto status : expectedStatuses) { + if (result == status) { + found = true; + break; + } + } + + if (!found) { + UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result)); + } + + return ev->Record; + } + + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { return TestGetExport(runtime, TTestTxConfig::SchemeShard, id, dbName, expectedStatuses); - } - - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus) { - return TestGetExport(runtime, schemeshardId, id, dbName, TVector<Ydb::StatusIds::StatusCode>(1, expectedStatus)); - } - - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus) { + return TestGetExport(runtime, schemeshardId, id, dbName, TVector<Ydb::StatusIds::StatusCode>(1, expectedStatus)); + } + + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus) { return TestGetExport(runtime, TTestTxConfig::SchemeShard, id, dbName, expectedStatus); - } - - TEvExport::TEvCancelExportRequest* CancelExportRequest(ui64 txId, const TString& dbName, ui64 exportId) { - return new TEvExport::TEvCancelExportRequest(txId, dbName, exportId); - } - - NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + TEvExport::TEvCancelExportRequest* CancelExportRequest(ui64 txId, const TString& dbName, ui64 exportId) { + return new TEvExport::TEvCancelExportRequest(txId, dbName, exportId); + } + + NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus) { ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), CancelExportRequest(txId, dbName, exportId)); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvExport::TEvCancelExportResponse>(handle); - UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); - - return ev->Record; - } - - NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus) { + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvExport::TEvCancelExportResponse>(handle); + UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); + + return ev->Record; + } + + NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus) { return TestCancelExport(runtime, TTestTxConfig::SchemeShard, txId, dbName, exportId, expectedStatus); - } - - TEvExport::TEvForgetExportRequest* ForgetExportRequest(ui64 txId, const TString& dbName, ui64 exportId) { - return new TEvExport::TEvForgetExportRequest(txId, dbName, exportId); - } - - void AsyncForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId) { + } + + TEvExport::TEvForgetExportRequest* ForgetExportRequest(ui64 txId, const TString& dbName, ui64 exportId) { + return new TEvExport::TEvForgetExportRequest(txId, dbName, exportId); + } + + void AsyncForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId) { ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), ForgetExportRequest(txId, dbName, exportId)); - } - - void AsyncForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId) { + } + + void AsyncForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId) { AsyncForgetExport(runtime, TTestTxConfig::SchemeShard, txId, dbName, exportId); - } - - NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus) { - AsyncForgetExport(runtime, schemeshardId, txId, dbName, exportId); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvExport::TEvForgetExportResponse>(handle); - UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); - - return ev->Record; - } - - NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus) { + AsyncForgetExport(runtime, schemeshardId, txId, dbName, exportId); + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvExport::TEvForgetExportResponse>(handle); + UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); + + return ev->Record; + } + + NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus) { return TestForgetExport(runtime, TTestTxConfig::SchemeShard, txId, dbName, exportId, expectedStatus); - } - - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr) { - NKikimrImport::TCreateImportRequest request; - UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); - + } + + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr) { + NKikimrImport::TCreateImportRequest request; + UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); + ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), new TEvImport::TEvCreateImportRequest(id, dbName, request)); - } - - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr) { + } + + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr) { AsyncImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr); - } - - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, - Ydb::StatusIds::StatusCode expectedStatus) { - AsyncImport(runtime, schemeshardId, id, dbName, requestStr); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCreateImportResponse>(handle); - UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); - } - - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + Ydb::StatusIds::StatusCode expectedStatus) { + AsyncImport(runtime, schemeshardId, id, dbName, requestStr); + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCreateImportResponse>(handle); + UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); + } + + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + Ydb::StatusIds::StatusCode expectedStatus) { TestImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, expectedStatus); - } - - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { + } + + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), new TEvImport::TEvGetImportRequest(dbName, id)); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvImport::TEvGetImportResponse>(handle); - const auto result = ev->Record.GetResponse().GetEntry().GetStatus(); - - bool found = false; - for (const auto status : expectedStatuses) { - if (result == status) { - found = true; - break; - } - } - - if (!found) { - UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result)); - } - - return ev->Record; - } - - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvImport::TEvGetImportResponse>(handle); + const auto result = ev->Record.GetResponse().GetEntry().GetStatus(); + + bool found = false; + for (const auto status : expectedStatuses) { + if (result == status) { + found = true; + break; + } + } + + if (!found) { + UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result)); + } + + return ev->Record; + } + + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses) { return TestGetImport(runtime, TTestTxConfig::SchemeShard, id, dbName, expectedStatuses); - } - - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus) { - return TestGetImport(runtime, schemeshardId, id, dbName, TVector<Ydb::StatusIds::StatusCode>(1, expectedStatus)); - } - - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus) { + return TestGetImport(runtime, schemeshardId, id, dbName, TVector<Ydb::StatusIds::StatusCode>(1, expectedStatus)); + } + + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus) { return TestGetImport(runtime, TTestTxConfig::SchemeShard, id, dbName, expectedStatus); - } - - TEvImport::TEvCancelImportRequest* CancelImportRequest(ui64 txId, const TString& dbName, ui64 importId) { - return new TEvImport::TEvCancelImportRequest(txId, dbName, importId); - } - - NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 importId, - Ydb::StatusIds::StatusCode expectedStatus) { + } + + TEvImport::TEvCancelImportRequest* CancelImportRequest(ui64 txId, const TString& dbName, ui64 importId) { + return new TEvImport::TEvCancelImportRequest(txId, dbName, importId); + } + + NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 importId, + Ydb::StatusIds::StatusCode expectedStatus) { ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), CancelImportRequest(txId, dbName, importId)); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCancelImportResponse>(handle); - UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); - - return ev->Record; - } - - NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 importId, - Ydb::StatusIds::StatusCode expectedStatus) { + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCancelImportResponse>(handle); + UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetStatus(), expectedStatus); + + return ev->Record; + } + + NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 importId, + Ydb::StatusIds::StatusCode expectedStatus) { return TestCancelImport(runtime, TTestTxConfig::SchemeShard, txId, dbName, importId, expectedStatus); - } - + } + NKikimrSchemeOp::TCreateSolomonVolume TakeTabletsFromAnotherSolomonVol(TString name, TString ls, ui32 count) { NKikimrSchemeOp::TCreateSolomonVolume volume; @@ -1366,14 +1366,14 @@ namespace NSchemeShardUT_Private { (let depth '('DepthLimit (Uint64 '%lu))) (let paths '('PathsLimit (Uint64 '%lu))) (let child '('ChildrenLimit (Uint64 '%lu))) - (let acl '('AclByteSizeLimit (Uint64 '%lu))) - (let columns '('TableColumnsLimit (Uint64 '%lu))) - (let colName '('TableColumnNameLengthLimit (Uint64 '%lu))) - (let keyCols '('TableKeyColumnsLimit (Uint64 '%lu))) - (let indices '('TableIndicesLimit (Uint64 '%lu))) + (let acl '('AclByteSizeLimit (Uint64 '%lu))) + (let columns '('TableColumnsLimit (Uint64 '%lu))) + (let colName '('TableColumnNameLengthLimit (Uint64 '%lu))) + (let keyCols '('TableKeyColumnsLimit (Uint64 '%lu))) + (let indices '('TableIndicesLimit (Uint64 '%lu))) (let shards '('ShardsLimit (Uint64 '%lu))) (let pathShards '('PathShardsLimit (Uint64 '%lu))) - (let consCopy '('ConsistentCopyingTargetsLimit (Uint64 '%lu))) + (let consCopy '('ConsistentCopyingTargetsLimit (Uint64 '%lu))) (let maxPathLength '('PathElementLength (Uint64 '%lu))) (let extraSymbols '('ExtraPathSymbolsAllowed (Utf8 '"%s"))) (let pqPartitions '('PQPartitionsLimit (Uint64 '%lu))) @@ -1525,26 +1525,26 @@ namespace NSchemeShardUT_Private { return { step, txId }; } - TEvIndexBuilder::TEvCreateRequest* CreateBuildIndexRequest(ui64 id, const TString& dbName, const TString& src, const TBuildIndexConfig& cfg) { + TEvIndexBuilder::TEvCreateRequest* CreateBuildIndexRequest(ui64 id, const TString& dbName, const TString& src, const TBuildIndexConfig& cfg) { NKikimrIndexBuilder::TIndexBuildSettings settings; settings.set_source_path(src); settings.set_max_batch_rows(2); settings.set_max_shards_in_flight(2); Ydb::Table::TableIndex& index = *settings.mutable_index(); - index.set_name(cfg.IndexName); - *index.mutable_index_columns() = {cfg.IndexColumns.begin(), cfg.IndexColumns.end()}; - *index.mutable_data_columns() = {cfg.DataColumns.begin(), cfg.DataColumns.end()}; + index.set_name(cfg.IndexName); + *index.mutable_index_columns() = {cfg.IndexColumns.begin(), cfg.IndexColumns.end()}; + *index.mutable_data_columns() = {cfg.DataColumns.begin(), cfg.DataColumns.end()}; - switch (cfg.IndexType) { + switch (cfg.IndexType) { case NKikimrSchemeOp::EIndexTypeGlobal: - *index.mutable_global_index() = Ydb::Table::GlobalIndex(); - break; + *index.mutable_global_index() = Ydb::Table::GlobalIndex(); + break; case NKikimrSchemeOp::EIndexTypeGlobalAsync: - *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; - default: - UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(cfg.IndexType)); + *index.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + break; + default: + UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(cfg.IndexType)); } return new TEvIndexBuilder::TEvCreateRequest(id, dbName, std::move(settings)); @@ -1558,25 +1558,25 @@ namespace NSchemeShardUT_Private { return result; } - void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg) { + void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg) { auto sender = runtime.AllocateEdgeActor(); - auto request = CreateBuildIndexRequest(id, dbName, src, cfg); + auto request = CreateBuildIndexRequest(id, dbName, src, cfg); ForwardToTablet(runtime, schemeShard, sender, request); } - void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, - const TString &src, const TString &name, TVector<TString> columns, TVector<TString> dataColumns) - { - AsyncBuilIndex(runtime, id, schemeShard, dbName, src, TBuildIndexConfig{ + void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, + const TString &src, const TString &name, TVector<TString> columns, TVector<TString> dataColumns) + { + AsyncBuilIndex(runtime, id, schemeShard, dbName, src, TBuildIndexConfig{ name, NKikimrSchemeOp::EIndexTypeGlobal, columns, dataColumns - }); - } - + }); + } + void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, - const TString &src, const TBuildIndexConfig& cfg, Ydb::StatusIds::StatusCode expectedStatus) + const TString &src, const TBuildIndexConfig& cfg, Ydb::StatusIds::StatusCode expectedStatus) { - AsyncBuilIndex(runtime, id, schemeShard, dbName, src, cfg); + AsyncBuilIndex(runtime, id, schemeShard, dbName, src, cfg); TAutoPtr<IEventHandle> handle; TEvIndexBuilder::TEvCreateResponse* event = runtime.GrabEdgeEvent<TEvIndexBuilder::TEvCreateResponse>(handle); @@ -1590,15 +1590,15 @@ namespace NSchemeShardUT_Private { << " issues was " << PrintIssues(event->Record.GetIssues())); } - void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, - const TString &src, const TString &name, TVector<TString> columns, - Ydb::StatusIds::StatusCode expectedStatus) - { - TestBuilIndex(runtime, id, schemeShard, dbName, src, TBuildIndexConfig{ + void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, + const TString &src, const TString &name, TVector<TString> columns, + Ydb::StatusIds::StatusCode expectedStatus) + { + TestBuilIndex(runtime, id, schemeShard, dbName, src, TBuildIndexConfig{ name, NKikimrSchemeOp::EIndexTypeGlobal, columns, {} - }, expectedStatus); - } - + }, expectedStatus); + } + TEvIndexBuilder::TEvCancelRequest* CreateCancelBuildIndexRequest( const ui64 id, const TString& dbName, const ui64 buildIndexId) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index 08684fe00a9..6b12c25534a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -1,19 +1,19 @@ #pragma once -#include "ls_checks.h" +#include "ls_checks.h" #include "test_env.h" -#include <library/cpp/testing/unittest/registar.h> - -#include <ydb/core/engine/mkql_engine_flat.h> -#include <ydb/core/protos/tx_datashard.pb.h> -#include <ydb/core/testlib/minikql_compile.h> -#include <ydb/core/tx/datashard/datashard.h> -#include <ydb/core/tx/schemeshard/schemeshard_build_index.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/core/engine/mkql_engine_flat.h> +#include <ydb/core/protos/tx_datashard.pb.h> +#include <ydb/core/testlib/minikql_compile.h> +#include <ydb/core/tx/datashard/datashard.h> +#include <ydb/core/tx/schemeshard/schemeshard_build_index.h> #include <ydb/core/tx/schemeshard/schemeshard_export.h> #include <ydb/core/tx/schemeshard/schemeshard_import.h> -#include <ydb/core/tx/schemeshard/schemeshard_types.h> -#include <ydb/library/yql/minikql/mkql_alloc.h> +#include <ydb/core/tx/schemeshard/schemeshard_types.h> +#include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> #include <util/stream/null.h> @@ -22,16 +22,16 @@ #undef Cdbg #if 1 - #define Cdbg Cerr + #define Cdbg Cerr #else - #define Cdbg Cnull + #define Cdbg Cnull #endif namespace NSchemeShardUT_Private { using namespace NKikimr; - using TEvTx = TEvSchemeShard::TEvModifySchemeTransaction; - + using TEvTx = TEvSchemeShard::TEvModifySchemeTransaction; + ////////// tablet NKikimrProto::EReplyStatus LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query, NKikimrMiniKQL::TResult& result, TString& err); NKikimrMiniKQL::TResult LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query); @@ -51,10 +51,10 @@ namespace NSchemeShardUT_Private { ////////// describe NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, const NKikimrSchemeOp::TDescribeOptions& opts); NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, const NKikimrSchemeOp::TDescribeOptions& opts); - NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false); - NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false); - NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false, bool showPrivate = false, bool returnBackups = false); - NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false, bool showPrivate = false, bool returnBackups = false); + NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false); + NKikimrScheme::TEvDescribeSchemeResult DescribePrivatePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false); + NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, ui64 schemeShard, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false, bool showPrivate = false, bool returnBackups = false); + NKikimrScheme::TEvDescribeSchemeResult DescribePath(TTestActorRuntime& runtime, const TString& path, bool returnPartitioning = false, bool returnBoundaries = false, bool showPrivate = false, bool returnBackups = false); TPathVersion ExtructPathVersion(const NKikimrScheme::TEvDescribeSchemeResult& describe); TPathVersion TestDescribeResult(const NKikimrScheme::TEvDescribeSchemeResult& describe, TVector<NLs::TCheckFunc> checks = {}); @@ -65,186 +65,186 @@ namespace NSchemeShardUT_Private { ////////// modification results void CheckExpected(const TVector<TEvSchemeShard::EStatus>& expected, TEvSchemeShard::EStatus result, const TString& reason); - void CheckExpected(const TVector<Ydb::StatusIds::StatusCode>& expected, Ydb::StatusIds::StatusCode result, const TString& reason); + void CheckExpected(const TVector<Ydb::StatusIds::StatusCode>& expected, Ydb::StatusIds::StatusCode result, const TString& reason); void TestModificationResult(TTestActorRuntime& runtime, ui64 txId,TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusAccepted); ui64 TestModificationResults(TTestActorRuntime& runtime, ui64 txId, const TVector<TEvSchemeShard::EStatus>& expectedResults); void SkipModificationReply(TTestActorRuntime& runtime, ui32 num = 1); - TEvTx* CombineSchemeTransactions(const TVector<TEvTx*>& transactions); - void AsyncSendTransaction(TTestActorRuntime& runtime, ui64 schemeShard, TEvTx* evTx); - - ////////// generic - - #define UT_GENERIC_PARAMS \ - const TString& parentPath, const TString& scheme - #define UT_PARAMS_BY_PATH_ID \ - ui64 pathId - - #define DEFINE_HELPERS(name, params, ...) \ - TEvTx* name##Request(ui64 schemeShardId, ui64 txId, params, __VA_ARGS__); \ - TEvTx* name##Request(ui64 txId, params, __VA_ARGS__); \ - void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, params, __VA_ARGS__); \ - void Async##name(TTestActorRuntime& runtime, ui64 txId, params, __VA_ARGS__); \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, params, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}, __VA_ARGS__); \ - ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, params, \ - const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}, __VA_ARGS__) - - #define GENERIC_HELPERS(name) DEFINE_HELPERS(name, UT_GENERIC_PARAMS, const TApplyIf& applyIf = {}) - #define GENERIC_WITH_ATTRS_HELPERS(name) DEFINE_HELPERS(name, UT_GENERIC_PARAMS, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs = {}, const TApplyIf& applyIf = {}) - #define DROP_BY_PATH_ID_HELPERS(name) DEFINE_HELPERS(name, UT_PARAMS_BY_PATH_ID, const TApplyIf& applyIf = {}) - - // subdomain - GENERIC_WITH_ATTRS_HELPERS(CreateSubDomain); - GENERIC_HELPERS(AlterSubDomain); - GENERIC_HELPERS(DropSubDomain); - GENERIC_HELPERS(ForceDropSubDomain); - ui64 TestCreateSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); - - // ext subdomain - GENERIC_WITH_ATTRS_HELPERS(CreateExtSubDomain); - GENERIC_WITH_ATTRS_HELPERS(AlterExtSubDomain); - GENERIC_HELPERS(ForceDropExtSubDomain); - ui64 TestCreateExtSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); - - // dir - GENERIC_WITH_ATTRS_HELPERS(MkDir); - GENERIC_HELPERS(RmDir); - DROP_BY_PATH_ID_HELPERS(ForceDropUnsafe); - - // user attrs - GENERIC_WITH_ATTRS_HELPERS(UserAttrs); + TEvTx* CombineSchemeTransactions(const TVector<TEvTx*>& transactions); + void AsyncSendTransaction(TTestActorRuntime& runtime, ui64 schemeShard, TEvTx* evTx); + + ////////// generic + + #define UT_GENERIC_PARAMS \ + const TString& parentPath, const TString& scheme + #define UT_PARAMS_BY_PATH_ID \ + ui64 pathId + + #define DEFINE_HELPERS(name, params, ...) \ + TEvTx* name##Request(ui64 schemeShardId, ui64 txId, params, __VA_ARGS__); \ + TEvTx* name##Request(ui64 txId, params, __VA_ARGS__); \ + void Async##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, params, __VA_ARGS__); \ + void Async##name(TTestActorRuntime& runtime, ui64 txId, params, __VA_ARGS__); \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, params, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}, __VA_ARGS__); \ + ui64 Test##name(TTestActorRuntime& runtime, ui64 txId, params, \ + const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}, __VA_ARGS__) + + #define GENERIC_HELPERS(name) DEFINE_HELPERS(name, UT_GENERIC_PARAMS, const TApplyIf& applyIf = {}) + #define GENERIC_WITH_ATTRS_HELPERS(name) DEFINE_HELPERS(name, UT_GENERIC_PARAMS, const NKikimrSchemeOp::TAlterUserAttributes& userAttrs = {}, const TApplyIf& applyIf = {}) + #define DROP_BY_PATH_ID_HELPERS(name) DEFINE_HELPERS(name, UT_PARAMS_BY_PATH_ID, const TApplyIf& applyIf = {}) + + // subdomain + GENERIC_WITH_ATTRS_HELPERS(CreateSubDomain); + GENERIC_HELPERS(AlterSubDomain); + GENERIC_HELPERS(DropSubDomain); + GENERIC_HELPERS(ForceDropSubDomain); + ui64 TestCreateSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); + + // ext subdomain + GENERIC_WITH_ATTRS_HELPERS(CreateExtSubDomain); + GENERIC_WITH_ATTRS_HELPERS(AlterExtSubDomain); + GENERIC_HELPERS(ForceDropExtSubDomain); + ui64 TestCreateExtSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& scheme, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); + + // dir + GENERIC_WITH_ATTRS_HELPERS(MkDir); + GENERIC_HELPERS(RmDir); + DROP_BY_PATH_ID_HELPERS(ForceDropUnsafe); + + // user attrs + GENERIC_WITH_ATTRS_HELPERS(UserAttrs); NKikimrSchemeOp::TAlterUserAttributes AlterUserAttrs(const TVector<std::pair<TString, TString>>& add, const TVector<TString>& drop = {}); - ui64 TestUserAttrs(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, - const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); - - // table - GENERIC_WITH_ATTRS_HELPERS(CreateTable); - GENERIC_HELPERS(CreateIndexedTable); - GENERIC_HELPERS(ConsistentCopyTables); - GENERIC_HELPERS(AlterTable); - GENERIC_HELPERS(SplitTable); - GENERIC_HELPERS(DropTable); - DROP_BY_PATH_ID_HELPERS(DropTable); - GENERIC_HELPERS(DropTableIndex); - - // backup & restore - GENERIC_HELPERS(BackupTable); - GENERIC_HELPERS(Restore); - - // cdc stream - GENERIC_HELPERS(CreateCdcStream); - GENERIC_HELPERS(AlterCdcStream); - GENERIC_HELPERS(DropCdcStream); - - // olap store - GENERIC_HELPERS(CreateOlapStore); - GENERIC_HELPERS(AlterOlapStore); - GENERIC_HELPERS(DropOlapStore); - DROP_BY_PATH_ID_HELPERS(DropOlapStore); - - // olap table - GENERIC_HELPERS(CreateOlapTable); - GENERIC_HELPERS(AlterOlapTable); - GENERIC_HELPERS(DropOlapTable); - DROP_BY_PATH_ID_HELPERS(DropOlapTable); - - // sequence - GENERIC_HELPERS(CreateSequence); - GENERIC_HELPERS(DropSequence); - DROP_BY_PATH_ID_HELPERS(DropSequence); - - // replication - GENERIC_HELPERS(CreateReplication); - GENERIC_HELPERS(DropReplication); - DROP_BY_PATH_ID_HELPERS(DropReplication); - - // pq - GENERIC_HELPERS(CreatePQGroup); - GENERIC_HELPERS(AlterPQGroup); - GENERIC_HELPERS(DropPQGroup); - DROP_BY_PATH_ID_HELPERS(DropPQGroup); - - // rtmr - GENERIC_HELPERS(CreateRtmrVolume); - - // solomon - GENERIC_HELPERS(CreateSolomon); - GENERIC_HELPERS(AlterSolomon); - GENERIC_HELPERS(DropSolomon); - DROP_BY_PATH_ID_HELPERS(DropSolomon); - NKikimrSchemeOp::TCreateSolomonVolume TakeTabletsFromAnotherSolomonVol(TString name, TString ls, ui32 count = 0); - - // kesus - GENERIC_HELPERS(CreateKesus); - GENERIC_HELPERS(AlterKesus); - GENERIC_HELPERS(DropKesus); - DROP_BY_PATH_ID_HELPERS(DropKesus); - - // filestore - GENERIC_HELPERS(CreateFileStore); - GENERIC_HELPERS(AlterFileStore); - GENERIC_HELPERS(DropFileStore); - DROP_BY_PATH_ID_HELPERS(DropFileStore); - - // nbs - GENERIC_HELPERS(CreateBlockStoreVolume); - GENERIC_HELPERS(AlterBlockStoreVolume); - GENERIC_HELPERS(DropBlockStoreVolume); - DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume); - void AsyncAssignBlockStoreVolume(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TString& mountToken, ui64 tokenVersion = 0); - void TestAssignBlockStoreVolume(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TString& mountToken, ui64 tokenVersion = 0, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusSuccess}); - - #undef DROP_BY_PATH_ID_HELPERS - #undef GENERIC_WITH_ATTRS_HELPERS - #undef GENERIC_HELPERS - #undef DEFINE_HELPERS - #undef UT_PARAMS_BY_PATH_ID - #undef UT_GENERIC_PARAMS - - ////////// non-generic - - // cancel tx - TEvSchemeShard::TEvCancelTx* CancelTxRequest(ui64 txId, ui64 targetTxId); - void AsyncCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId); - void TestCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId, - const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); - - // modify acl - TEvTx* CreateModifyACLRequest(ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); - void AsyncModifyACL(TTestActorRuntime& runtime, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); - void AsyncModifyACL(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); - void TestModifyACL(TTestActorRuntime& runtime, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusSuccess); - void TestModifyACL(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusSuccess); - - // upgrade subdomain - TEvTx* UpgradeSubDomainRequest(ui64 txId, const TString& parentPath, const TString& name); - void AsyncUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name); - void TestUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus>& expectedResults); - void TestUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name); - - TEvTx* UpgradeSubDomainDecisionRequest(ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); - void AsyncUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); - void TestUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus>& expectedResults, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); - void TestUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); - - // copy table - TEvTx* CopyTableRequest(ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName, TApplyIf applyIf = {}); + ui64 TestUserAttrs(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, + const NKikimrSchemeOp::TAlterUserAttributes& userAttrs); + + // table + GENERIC_WITH_ATTRS_HELPERS(CreateTable); + GENERIC_HELPERS(CreateIndexedTable); + GENERIC_HELPERS(ConsistentCopyTables); + GENERIC_HELPERS(AlterTable); + GENERIC_HELPERS(SplitTable); + GENERIC_HELPERS(DropTable); + DROP_BY_PATH_ID_HELPERS(DropTable); + GENERIC_HELPERS(DropTableIndex); + + // backup & restore + GENERIC_HELPERS(BackupTable); + GENERIC_HELPERS(Restore); + + // cdc stream + GENERIC_HELPERS(CreateCdcStream); + GENERIC_HELPERS(AlterCdcStream); + GENERIC_HELPERS(DropCdcStream); + + // olap store + GENERIC_HELPERS(CreateOlapStore); + GENERIC_HELPERS(AlterOlapStore); + GENERIC_HELPERS(DropOlapStore); + DROP_BY_PATH_ID_HELPERS(DropOlapStore); + + // olap table + GENERIC_HELPERS(CreateOlapTable); + GENERIC_HELPERS(AlterOlapTable); + GENERIC_HELPERS(DropOlapTable); + DROP_BY_PATH_ID_HELPERS(DropOlapTable); + + // sequence + GENERIC_HELPERS(CreateSequence); + GENERIC_HELPERS(DropSequence); + DROP_BY_PATH_ID_HELPERS(DropSequence); + + // replication + GENERIC_HELPERS(CreateReplication); + GENERIC_HELPERS(DropReplication); + DROP_BY_PATH_ID_HELPERS(DropReplication); + + // pq + GENERIC_HELPERS(CreatePQGroup); + GENERIC_HELPERS(AlterPQGroup); + GENERIC_HELPERS(DropPQGroup); + DROP_BY_PATH_ID_HELPERS(DropPQGroup); + + // rtmr + GENERIC_HELPERS(CreateRtmrVolume); + + // solomon + GENERIC_HELPERS(CreateSolomon); + GENERIC_HELPERS(AlterSolomon); + GENERIC_HELPERS(DropSolomon); + DROP_BY_PATH_ID_HELPERS(DropSolomon); + NKikimrSchemeOp::TCreateSolomonVolume TakeTabletsFromAnotherSolomonVol(TString name, TString ls, ui32 count = 0); + + // kesus + GENERIC_HELPERS(CreateKesus); + GENERIC_HELPERS(AlterKesus); + GENERIC_HELPERS(DropKesus); + DROP_BY_PATH_ID_HELPERS(DropKesus); + + // filestore + GENERIC_HELPERS(CreateFileStore); + GENERIC_HELPERS(AlterFileStore); + GENERIC_HELPERS(DropFileStore); + DROP_BY_PATH_ID_HELPERS(DropFileStore); + + // nbs + GENERIC_HELPERS(CreateBlockStoreVolume); + GENERIC_HELPERS(AlterBlockStoreVolume); + GENERIC_HELPERS(DropBlockStoreVolume); + DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume); + void AsyncAssignBlockStoreVolume(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TString& mountToken, ui64 tokenVersion = 0); + void TestAssignBlockStoreVolume(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TString& mountToken, ui64 tokenVersion = 0, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusSuccess}); + + #undef DROP_BY_PATH_ID_HELPERS + #undef GENERIC_WITH_ATTRS_HELPERS + #undef GENERIC_HELPERS + #undef DEFINE_HELPERS + #undef UT_PARAMS_BY_PATH_ID + #undef UT_GENERIC_PARAMS + + ////////// non-generic + + // cancel tx + TEvSchemeShard::TEvCancelTx* CancelTxRequest(ui64 txId, ui64 targetTxId); + void AsyncCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId); + void TestCancelTxTable(TTestActorRuntime& runtime, ui64 txId, ui64 targetTxId, + const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); + + // modify acl + TEvTx* CreateModifyACLRequest(ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); + void AsyncModifyACL(TTestActorRuntime& runtime, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); + void AsyncModifyACL(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner); + void TestModifyACL(TTestActorRuntime& runtime, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusSuccess); + void TestModifyACL(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, TString parentPath, TString name, const TString& diffAcl, const TString& newOwner, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusSuccess); + + // upgrade subdomain + TEvTx* UpgradeSubDomainRequest(ui64 txId, const TString& parentPath, const TString& name); + void AsyncUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name); + void TestUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus>& expectedResults); + void TestUpgradeSubDomain(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name); + + TEvTx* UpgradeSubDomainDecisionRequest(ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); + void AsyncUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); + void TestUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus>& expectedResults, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); + void TestUpgradeSubDomainDecision(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, NKikimrSchemeOp::TUpgradeSubDomain::EDecision taskType); + + // copy table + TEvTx* CopyTableRequest(ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName, TApplyIf applyIf = {}); void AsyncCopyTable(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName); void AsyncCopyTable(TTestActorRuntime& runtime, ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName); void TestCopyTable(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusAccepted); void TestCopyTable(TTestActorRuntime& runtime, ui64 txId, const TString& dstPath, const TString& dstName, const TString& srcFullName, TEvSchemeShard::EStatus expectedResult = NKikimrScheme::StatusAccepted); - // move table + // move table TEvTx* MoveTableRequest(ui64 txId, const TString& srcPath, const TString& dstPath, ui64 schemeShard = TTestTxConfig::SchemeShard, const TApplyIf& applyIf = {}); void AsyncMoveTable(TTestActorRuntime& runtime, ui64 txId, const TString& srcPath, const TString& dstPath, ui64 schemeShard = TTestTxConfig::SchemeShard); void TestMoveTable(TTestActorRuntime& runtime, ui64 txId, const TString& srcMove, const TString& dstMove, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); void TestMoveTable(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, const TString& srcMove, const TString& dstMove, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); - // locks - TEvTx* LockRequest(ui64 txId, const TString &parentPath, const TString& name); + // locks + TEvTx* LockRequest(ui64 txId, const TString &parentPath, const TString& name); void AsyncLock(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, const TString& parentPath, const TString& name); void AsyncLock(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name); void TestLock(TTestActorRuntime& runtime, ui64 schemeShard, ui64 txId, const TString& parentPath, const TString& name, @@ -258,77 +258,77 @@ namespace NSchemeShardUT_Private { void TestUnlock(TTestActorRuntime& runtime, ui64 txId, ui64 lockId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus> expectedResults = {NKikimrScheme::StatusAccepted}); - // index build - struct TBuildIndexConfig { - TString IndexName; - NKikimrSchemeOp::EIndexType IndexType = NKikimrSchemeOp::EIndexTypeGlobal; - TVector<TString> IndexColumns; - TVector<TString> DataColumns; - }; - - TEvIndexBuilder::TEvCreateRequest* CreateBuildIndexRequest(ui64 id, const TString& dbName, const TString& src, const TBuildIndexConfig& cfg); - void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg); - void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TString &name, TVector<TString> columns, TVector<TString> dataColumns = {}); - void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TString &name, TVector<TString> columns, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - TEvIndexBuilder::TEvCancelRequest* CreateCancelBuildIndexRequest(const ui64 id, const TString& dbName, const ui64 buildIndexId); - NKikimrIndexBuilder::TEvCancelResponse TestCancelBuildIndex(TTestActorRuntime& runtime, const ui64 id, const ui64 schemeShard, const TString &dbName, const ui64 buildIndexId, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); - TEvIndexBuilder::TEvListRequest* ListBuildIndexRequest(const TString& dbName); - NKikimrIndexBuilder::TEvListResponse TestListBuilIndex(TTestActorRuntime& runtime, ui64 schemeShard, const TString &dbName); - TEvIndexBuilder::TEvGetRequest* GetBuildIndexRequest(const TString& dbName, ui64 id); - NKikimrIndexBuilder::TEvGetResponse TestGetBuilIndex(TTestActorRuntime& runtime, ui64 schemeShard, const TString &dbName, ui64 id); - TEvIndexBuilder::TEvForgetRequest* ForgetBuildIndexRequest(const ui64 id, const TString &dbName, const ui64 buildIndexId); - NKikimrIndexBuilder::TEvForgetResponse TestForgetBuilIndex(TTestActorRuntime& runtime, const ui64 id, const ui64 schemeShard, const TString &dbName, const ui64 buildIndexId, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - - ////////// export - void AsyncExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); - void AsyncExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); - void TestExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - void TestExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - TEvExport::TEvCancelExportRequest* CancelExportRequest(ui64 txId, const TString& dbName, ui64 exportId); - NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - TEvExport::TEvForgetExportRequest* ForgetExportRequest(ui64 txId, const TString& dbName, ui64 exportId); - void AsyncForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId); - void AsyncForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId); - NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - - ////////// import - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr); - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr); - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - TEvImport::TEvCancelImportRequest* CancelImportRequest(ui64 txId, const TString& dbName, ui64 importId); - NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 importId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 importId, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - + // index build + struct TBuildIndexConfig { + TString IndexName; + NKikimrSchemeOp::EIndexType IndexType = NKikimrSchemeOp::EIndexTypeGlobal; + TVector<TString> IndexColumns; + TVector<TString> DataColumns; + }; + + TEvIndexBuilder::TEvCreateRequest* CreateBuildIndexRequest(ui64 id, const TString& dbName, const TString& src, const TBuildIndexConfig& cfg); + void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg); + void AsyncBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TString &name, TVector<TString> columns, TVector<TString> dataColumns = {}); + void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TBuildIndexConfig &cfg, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + void TestBuilIndex(TTestActorRuntime& runtime, ui64 id, ui64 schemeShard, const TString &dbName, const TString &src, const TString &name, TVector<TString> columns, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + TEvIndexBuilder::TEvCancelRequest* CreateCancelBuildIndexRequest(const ui64 id, const TString& dbName, const ui64 buildIndexId); + NKikimrIndexBuilder::TEvCancelResponse TestCancelBuildIndex(TTestActorRuntime& runtime, const ui64 id, const ui64 schemeShard, const TString &dbName, const ui64 buildIndexId, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); + TEvIndexBuilder::TEvListRequest* ListBuildIndexRequest(const TString& dbName); + NKikimrIndexBuilder::TEvListResponse TestListBuilIndex(TTestActorRuntime& runtime, ui64 schemeShard, const TString &dbName); + TEvIndexBuilder::TEvGetRequest* GetBuildIndexRequest(const TString& dbName, ui64 id); + NKikimrIndexBuilder::TEvGetResponse TestGetBuilIndex(TTestActorRuntime& runtime, ui64 schemeShard, const TString &dbName, ui64 id); + TEvIndexBuilder::TEvForgetRequest* ForgetBuildIndexRequest(const ui64 id, const TString &dbName, const ui64 buildIndexId); + NKikimrIndexBuilder::TEvForgetResponse TestForgetBuilIndex(TTestActorRuntime& runtime, const ui64 id, const ui64 schemeShard, const TString &dbName, const ui64 buildIndexId, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + + ////////// export + void AsyncExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void AsyncExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void TestExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + void TestExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrExport::TEvGetExportResponse TestGetExport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + TEvExport::TEvCancelExportRequest* CancelExportRequest(ui64 txId, const TString& dbName, ui64 exportId); + NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrExport::TEvCancelExportResponse TestCancelExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + TEvExport::TEvForgetExportRequest* ForgetExportRequest(ui64 txId, const TString& dbName, ui64 exportId); + void AsyncForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId); + void AsyncForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId); + NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrExport::TEvForgetExportResponse TestForgetExport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 exportId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + + ////////// import + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr); + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr); + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + TEvImport::TEvCancelImportRequest* CancelImportRequest(ui64 txId, const TString& dbName, ui64 importId); + NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 txId, const TString& dbName, ui64 importId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + NKikimrImport::TEvCancelImportResponse TestCancelImport(TTestActorRuntime& runtime, ui64 txId, const TString& dbName, ui64 importId, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); + ////////// datashard ui64 GetDatashardState(TTestActorRuntime& runtime, ui64 tabletId); TString SetAllowLogBatching(TTestActorRuntime& runtime, ui64 tabletId, bool v); @@ -352,40 +352,40 @@ namespace NSchemeShardUT_Private { NLs::TCheckFunc ShardsIsReady(TTestActorRuntime& runtime); - template <typename TCreateFunc> - void CreateWithIntermediateDirs(TCreateFunc func) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - func(runtime, ++t.TxId, "/MyRoot", false); // invalid - func(runtime, ++t.TxId, "/MyRoot", true); // valid - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - { - TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Valid/x/y/z"), - {NLs::Finished}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Invalid"), - {NLs::PathNotExist}); - } - }); - } - - template <typename TCreateFunc> - void CreateWithIntermediateDirsForceDrop(TCreateFunc func) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - func(runtime, ++t.TxId, "/MyRoot"); - AsyncForceDropUnsafe(runtime, ++t.TxId, 3); - t.TestEnv->TestWaitNotification(runtime, {t.TxId - 1, t.TxId}); - - { - TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "/MyRoot/x"), - {NLs::PathNotExist}); - } - }); - } - + template <typename TCreateFunc> + void CreateWithIntermediateDirs(TCreateFunc func) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + func(runtime, ++t.TxId, "/MyRoot", false); // invalid + func(runtime, ++t.TxId, "/MyRoot", true); // valid + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/Valid/x/y/z"), + {NLs::Finished}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/Invalid"), + {NLs::PathNotExist}); + } + }); + } + + template <typename TCreateFunc> + void CreateWithIntermediateDirsForceDrop(TCreateFunc func) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + func(runtime, ++t.TxId, "/MyRoot"); + AsyncForceDropUnsafe(runtime, ++t.TxId, 3); + t.TestEnv->TestWaitNotification(runtime, {t.TxId - 1, t.TxId}); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/x"), + {NLs::PathNotExist}); + } + }); + } + TRowVersion CreateVolatileSnapshot( TTestActorRuntime& runtime, const TVector<TString>& tables, @@ -399,7 +399,7 @@ namespace NSchemeShardUT_Private { NKikimrScheme::TEvFindTabletSubDomainPathIdResult::EStatus expected = NKikimrScheme::TEvFindTabletSubDomainPathIdResult::SUCCESS); // Login - TEvTx* CreateAlterLoginCreateUser(ui64 txId, const TString& user, const TString& password); + TEvTx* CreateAlterLoginCreateUser(ui64 txId, const TString& user, const TString& password); NKikimrScheme::TEvLoginResult Login(TTestActorRuntime& runtime, const TString& user, const TString& password); // Mimics data query to a single table with multiple partitions diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index cc587c1ef7c..c5d6c5a973a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -42,44 +42,44 @@ void InSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record) { } } -TCheckFunc IsSubDomain(const TString& name) { +TCheckFunc IsSubDomain(const TString& name) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& selfPath = pathDescr.GetSelf(); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetName(), name); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeSubDomain); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetName(), name); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeSubDomain); }; } -TCheckFunc IsExternalSubDomain(const TString& name) { +TCheckFunc IsExternalSubDomain(const TString& name) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& selfPath = pathDescr.GetSelf(); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetName(), name); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetName(), name); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); }; } -bool IsGoodDomainStatus(NKikimrScheme::EStatus status) { - switch (status) { - case NKikimrScheme::StatusSuccess: - case NKikimrScheme::StatusRedirectDomain: - return true; - default: - return false; - } -} - -TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShardId) { +bool IsGoodDomainStatus(NKikimrScheme::EStatus status) { + switch (status) { + case NKikimrScheme::StatusSuccess: + case NKikimrScheme::StatusRedirectDomain: + return true; + default: + return false; + } +} + +TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShardId) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); if (record.GetStatus() == NKikimrScheme::StatusSuccess) { const auto& pathDescr = record.GetPathDescription(); const auto& selfPath = pathDescr.GetSelf(); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); UNIT_ASSERT(pathDescr.HasDomainDescription()); const auto& domainDesc = pathDescr.GetDomainDescription(); UNIT_ASSERT(domainDesc.HasProcessingParams()); @@ -96,9 +96,9 @@ TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShardId) { }; } -TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVPId) { +TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVPId) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); UNIT_ASSERT(pathDescr.HasDomainDescription()); const auto& domainDesc = pathDescr.GetDomainDescription(); @@ -121,13 +121,13 @@ void InExternalSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_UNEQUAL(domain.GetDomainKey().GetPathId(), 1); UNIT_ASSERT(domain.GetDomainKey().HasSchemeShard()); - const auto& extDomainPath = record.GetLastExistedPrefixDescription(); - UNIT_ASSERT_VALUES_EQUAL(extDomainPath.GetSelf().GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); + const auto& extDomainPath = record.GetLastExistedPrefixDescription(); + UNIT_ASSERT_VALUES_EQUAL(extDomainPath.GetSelf().GetPathType(), NKikimrSchemeOp::EPathTypeExtSubDomain); } -TCheckFunc SubDomainVersion(ui64 descrVersion) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); +TCheckFunc SubDomainVersion(ui64 descrVersion) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& processingParams = pathDescr.GetDomainDescription().GetProcessingParams(); UNIT_ASSERT_EQUAL_C(processingParams.GetVersion(), descrVersion, @@ -139,25 +139,25 @@ TCheckFunc SubDomainVersion(ui64 descrVersion) { }; } -TCheckFunc DomainKey(ui64 pathId, ui64 schemeshardId) { +TCheckFunc DomainKey(ui64 pathId, ui64 schemeshardId) { return DomainKey(TPathId(schemeshardId, pathId)); } -TCheckFunc DomainKey(TPathId pathId) { +TCheckFunc DomainKey(TPathId pathId) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); const auto& pathDescr = record.GetPathDescription(); - const auto& domainKey = pathDescr.GetDomainDescription().GetDomainKey(); + const auto& domainKey = pathDescr.GetDomainDescription().GetDomainKey(); - UNIT_ASSERT_VALUES_EQUAL(domainKey.GetPathId(), pathId.LocalPathId); - UNIT_ASSERT_VALUES_EQUAL(domainKey.GetSchemeShard(), pathId.OwnerId); + UNIT_ASSERT_VALUES_EQUAL(domainKey.GetPathId(), pathId.LocalPathId); + UNIT_ASSERT_VALUES_EQUAL(domainKey.GetSchemeShard(), pathId.OwnerId); }; } -TCheckFunc StoragePoolsEqual(TSet<TString> poolNames) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); +TCheckFunc StoragePoolsEqual(TSet<TString> poolNames) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); TSet<TString> presentPools; @@ -170,56 +170,56 @@ TCheckFunc StoragePoolsEqual(TSet<TString> poolNames) { }; } -TCheckFunc DomainCoordinators(TVector<ui64> coordinators) { +TCheckFunc DomainCoordinators(TVector<ui64> coordinators) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); const auto& pathDescr = record.GetPathDescription(); const auto& processingParams = pathDescr.GetDomainDescription().GetProcessingParams(); - UNIT_ASSERT_VALUES_EQUAL(processingParams.CoordinatorsSize(), coordinators.size()); + UNIT_ASSERT_VALUES_EQUAL(processingParams.CoordinatorsSize(), coordinators.size()); TVector<ui64> actual(processingParams.GetCoordinators().begin(),processingParams.GetCoordinators().end()); UNIT_ASSERT_EQUAL(actual, coordinators); }; } -TCheckFunc DomainMediators(TVector<ui64> mediators) { +TCheckFunc DomainMediators(TVector<ui64> mediators) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); const auto& pathDescr = record.GetPathDescription(); const auto& processingParams = pathDescr.GetDomainDescription().GetProcessingParams(); - UNIT_ASSERT_VALUES_EQUAL(processingParams.MediatorsSize(), mediators.size()); + UNIT_ASSERT_VALUES_EQUAL(processingParams.MediatorsSize(), mediators.size()); TVector<ui64> actual(processingParams.GetMediators().begin(), processingParams.GetMediators().end()); UNIT_ASSERT_EQUAL(actual, mediators); }; } -TCheckFunc DomainSchemeshard(ui64 schemeshard) { +TCheckFunc DomainSchemeshard(ui64 schemeshard) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); const auto& pathDescr = record.GetPathDescription(); const auto& processingParams = pathDescr.GetDomainDescription().GetProcessingParams(); if (schemeshard) { - UNIT_ASSERT_VALUES_EQUAL(processingParams.GetSchemeShard(), schemeshard); + UNIT_ASSERT_VALUES_EQUAL(processingParams.GetSchemeShard(), schemeshard); } else { UNIT_ASSERT(!processingParams.HasSchemeShard()); } }; } -TCheckFunc DomainSettings(ui32 planResolution, ui32 timeCastBucketsPerMediator) { +TCheckFunc DomainSettings(ui32 planResolution, ui32 timeCastBucketsPerMediator) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& processingParams = pathDescr.GetDomainDescription().GetProcessingParams(); - UNIT_ASSERT_VALUES_EQUAL(processingParams.GetPlanResolution(), planResolution); - UNIT_ASSERT_VALUES_EQUAL(processingParams.GetTimeCastBucketsPerMediator(), timeCastBucketsPerMediator); + UNIT_ASSERT_VALUES_EQUAL(processingParams.GetPlanResolution(), planResolution); + UNIT_ASSERT_VALUES_EQUAL(processingParams.GetTimeCastBucketsPerMediator(), timeCastBucketsPerMediator); }; } @@ -249,7 +249,7 @@ void NotFinished(const NKikimrScheme::TEvDescribeSchemeResult& record) { } } } - + void Finished(const NKikimrScheme::TEvDescribeSchemeResult& record) { PathExist(record); @@ -263,7 +263,7 @@ void Finished(const NKikimrScheme::TEvDescribeSchemeResult& record) { TCheckFunc ExtractVolumeConfig(NKikimrBlockStore::TVolumeConfig* config) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); UNIT_ASSERT(record.HasPathDescription()); const auto& descr = record.GetPathDescription(); @@ -277,7 +277,7 @@ TCheckFunc ExtractVolumeConfig(NKikimrBlockStore::TVolumeConfig* config) { TCheckFunc CheckMountToken(const TString& name, const TString& expectedOwner) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); UNIT_ASSERT(record.HasPathDescription()); const auto& descr = record.GetPathDescription(); @@ -294,8 +294,8 @@ TCheckFunc CheckMountToken(const TString& name, const TString& expectedOwner) { } TCheckFunc UserAttrsEqual(TUserAttrs attrs) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); TUserAttrs requered = attrs; @@ -330,8 +330,8 @@ TCheckFunc UserAttrsEqual(TUserAttrs attrs) { } TCheckFunc UserAttrsHas(TUserAttrs attrs) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); TUserAttrs requered = attrs; @@ -355,10 +355,10 @@ TCheckFunc UserAttrsHas(TUserAttrs attrs) { } void IsTable(const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& selfPath = pathDescr.GetSelf(); - UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeTable); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeTable); } TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const TSet<TString>& droppedColumns, const TSet<TString> keyColumns, @@ -466,16 +466,16 @@ TCheckFunc CheckPartCount(const TString& name, ui32 partCount, ui32 maxParts, ui UNIT_ASSERT_VALUES_EQUAL(partCount, pqs.size()); UNIT_ASSERT_VALUES_EQUAL(tabletCount, tablets.size()); - if (!pqs.empty()) { + if (!pqs.empty()) { UNIT_ASSERT_VALUES_EQUAL(*pqs.begin(), 0); - UNIT_ASSERT_VALUES_EQUAL(*pqs.rbegin(), partCount - 1); + UNIT_ASSERT_VALUES_EQUAL(*pqs.rbegin(), partCount - 1); } }; } TCheckFunc PathVersionEqual(ui64 version) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& self = pathDescr.GetSelf(); const auto& curVersion = self.GetPathVersion(); @@ -488,8 +488,8 @@ TCheckFunc PathVersionEqual(ui64 version) { } TCheckFunc PathVersionOneOf(TSet<ui64> versions) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& self = pathDescr.GetSelf(); const auto& curVersion = self.GetPathVersion(); @@ -502,35 +502,35 @@ TCheckFunc PathVersionOneOf(TSet<ui64> versions) { } TCheckFunc PathIdEqual(ui64 pathId) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& self = pathDescr.GetSelf(); const auto& curId = self.GetPathId(); - UNIT_ASSERT_VALUES_EQUAL(curId, pathId); + UNIT_ASSERT_VALUES_EQUAL(curId, pathId); }; } TCheckFunc PathIdEqual(TPathId pathId) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& self = pathDescr.GetSelf(); const auto& curId = TPathId(self.GetSchemeshardId(), self.GetPathId()); - UNIT_ASSERT_VALUES_EQUAL(curId, pathId); + UNIT_ASSERT_VALUES_EQUAL(curId, pathId); }; } TCheckFunc PathStringEqual(const TString& expected) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPath(), expected); + UNIT_ASSERT_VALUES_EQUAL(record.GetPath(), expected); }; } TCheckFunc PathsInsideDomain(ui64 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); - + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); const auto& curCount = domain.GetPathsInside(); @@ -543,9 +543,9 @@ TCheckFunc PathsInsideDomain(ui64 count) { } TCheckFunc PQPartitionsInsideDomain(ui64 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); - + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); const auto& curCount = domain.GetPQPartitionsInside(); @@ -557,9 +557,9 @@ TCheckFunc PQPartitionsInsideDomain(ui64 count) { }; } -TCheckFunc PathsInsideDomainOneOf(TSet<ui64> variants) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); +TCheckFunc PathsInsideDomainOneOf(TSet<ui64> variants) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); @@ -573,9 +573,9 @@ TCheckFunc PathsInsideDomainOneOf(TSet<ui64> variants) { } TCheckFunc ShardsInsideDomain(ui64 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); - + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); const auto& curCount = domain.GetShardsInside(); @@ -588,9 +588,9 @@ TCheckFunc ShardsInsideDomain(ui64 count) { } TCheckFunc ShardsInsideDomainOneOf(TSet<ui64> variants) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); - + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); const auto& curCount = domain.GetShardsInside(); @@ -603,8 +603,8 @@ TCheckFunc ShardsInsideDomainOneOf(TSet<ui64> variants) { } TCheckFunc DomainLimitsIs(ui64 maxPaths, ui64 maxShards, ui64 maxPQPartitions) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); const auto& pathLimit = domain.GetPathsLimit(); @@ -629,54 +629,54 @@ TCheckFunc DomainLimitsIs(ui64 maxPaths, ui64 maxShards, ui64 maxPQPartitions) { } TCheckFunc FreezeStateEqual(NKikimrSchemeOp::EFreezeState expectedState) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); UNIT_ASSERT(record.HasPathDescription()); UNIT_ASSERT(record.GetPathDescription().HasTable()); UNIT_ASSERT(record.GetPathDescription().GetTable().HasPartitionConfig()); UNIT_ASSERT(record.GetPathDescription().GetTable().GetPartitionConfig().HasFreezeState()); - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetFreezeState(), expectedState); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetFreezeState(), expectedState); }; } TCheckFunc ChildrenCount(ui32 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().ChildrenSize(), count); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().ChildrenSize(), count); }; } -TCheckFunc IndexesCount(ui32 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().TableIndexesSize(), count); +TCheckFunc IndexesCount(ui32 count) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().TableIndexesSize(), count); }; } -TCheckFunc IndexType(NKikimrSchemeOp::EIndexType type) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetType(), type); +TCheckFunc IndexType(NKikimrSchemeOp::EIndexType type) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetType(), type); }; } -TCheckFunc IndexState(NKikimrSchemeOp::EIndexState state) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetState(), state); +TCheckFunc IndexState(NKikimrSchemeOp::EIndexState state) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetState(), state); }; } -TCheckFunc IndexKeys(const TVector<TString>& keyNames) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().KeyColumnNamesSize(), keyNames.size()); +TCheckFunc IndexKeys(const TVector<TString>& keyNames) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().KeyColumnNamesSize(), keyNames.size()); for (ui32 keyId = 0; keyId < keyNames.size(); ++keyId) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetKeyColumnNames(keyId), keyNames.at(keyId)); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetKeyColumnNames(keyId), keyNames.at(keyId)); } }; } -TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().DataColumnNamesSize(), dataColumnNames.size()); +TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().DataColumnNamesSize(), dataColumnNames.size()); for (ui32 colId = 0; colId < dataColumnNames.size(); ++colId) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetDataColumnNames(colId), dataColumnNames.at(colId)); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTableIndex().GetDataColumnNames(colId), dataColumnNames.at(colId)); } }; } @@ -686,35 +686,35 @@ void NoChildren(const NKikimrScheme::TEvDescribeSchemeResult& record) { } void PathNotExist(const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusPathDoesNotExist); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusPathDoesNotExist); } void PathExist(const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); } void PathRedirected(const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusRedirectDomain); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusRedirectDomain); } TCheckFunc CreatedAt(ui64 txId) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetSelf().GetCreateTxId(), txId); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetSelf().GetCreateTxId(), txId); }; } TCheckFunc PartitionCount(ui32 count) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().TablePartitionsSize(), count); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().TablePartitionsSize(), count); }; } TCheckFunc FollowerCount(ui32 count) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetFollowerCount(), count); - }; -} - + }; +} + TCheckFunc CrossDataCenterFollowerCount(ui32 count) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetCrossDataCenterFollowerCount(), count); @@ -760,49 +760,49 @@ TCheckFunc SizeToSplitEqual(ui32 size) { }; } -TCheckFunc MinPartitionsCountEqual(ui32 count) { +TCheckFunc MinPartitionsCountEqual(ui32 count) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetMinPartitionsCount(), count); - }; -} - -TCheckFunc MaxPartitionsCountEqual(ui32 count) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetMinPartitionsCount(), count); + }; +} + +TCheckFunc MaxPartitionsCountEqual(ui32 count) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetMaxPartitionsCount(), count); - }; -} - -TCheckFunc PartitioningByLoadStatus(bool status) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetMaxPartitionsCount(), count); + }; +} + +TCheckFunc PartitioningByLoadStatus(bool status) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetSplitByLoadSettings().GetEnabled(), status); - }; -} - + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetPartitioningPolicy().GetSplitByLoadSettings().GetEnabled(), status); + }; +} + void NoBackupInFly(const NKikimrScheme::TEvDescribeSchemeResult &record) { UNIT_ASSERT(!record.GetPathDescription().HasBackupProgress()); } TCheckFunc HasBackupInFly(ui64 txId) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT(record.GetPathDescription().HasBackupProgress()); - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetBackupProgress().GetTxId(), txId); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetBackupProgress().GetTxId(), txId); }; } TCheckFunc BackupHistoryCount(ui64 count) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().LastBackupResultSize(), count); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().LastBackupResultSize(), count); }; } TCheckFunc ColumnFamiliesCount(ui32 size) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().ColumnFamiliesSize(), size); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().ColumnFamiliesSize(), size); }; } TCheckFunc ColumnFamiliesHas(ui32 familyId) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { bool has = false; for (const auto& x: record.GetPathDescription().GetTable().GetPartitionConfig().GetColumnFamilies()) { if (x.GetId() == familyId) { @@ -815,7 +815,7 @@ TCheckFunc ColumnFamiliesHas(ui32 familyId) { } TCheckFunc ColumnFamiliesHas(ui32 familyId, const TString& familyName) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { bool has = false; for (const auto& x: record.GetPathDescription().GetTable().GetPartitionConfig().GetColumnFamilies()) { if (x.GetId() == familyId && x.GetName() == familyName) { @@ -827,35 +827,35 @@ TCheckFunc ColumnFamiliesHas(ui32 familyId, const TString& familyName) { }; } -TCheckFunc KeyBloomFilterStatus(bool status) { +TCheckFunc KeyBloomFilterStatus(bool status) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetEnableFilterByKey(), status); - }; -} - + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetTable().GetPartitionConfig().GetEnableFilterByKey(), status); + }; +} + TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter, NKikimrSchemeOp::TTTLSettings::EUnit columnUnit) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& ttl = record.GetPathDescription().GetTable().GetTTLSettings(); - UNIT_ASSERT(ttl.HasEnabled()); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnUnit(), columnUnit); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), expireAfter.Seconds()); - }; -} - -TCheckFunc HasTtlDisabled() { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& ttl = record.GetPathDescription().GetTable().GetTTLSettings(); - UNIT_ASSERT(ttl.HasDisabled()); - }; -} - -TCheckFunc IsBackupTable(bool value) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(value, record.GetPathDescription().GetTable().GetIsBackup()); - }; -} - + const auto& ttl = record.GetPathDescription().GetTable().GetTTLSettings(); + UNIT_ASSERT(ttl.HasEnabled()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnUnit(), columnUnit); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), expireAfter.Seconds()); + }; +} + +TCheckFunc HasTtlDisabled() { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + const auto& ttl = record.GetPathDescription().GetTable().GetTTLSettings(); + UNIT_ASSERT(ttl.HasDisabled()); + }; +} + +TCheckFunc IsBackupTable(bool value) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(value, record.GetPathDescription().GetTable().GetIsBackup()); + }; +} + TCheckFunc HasOlapTableSchemaPreset(const TString& presetName) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { const auto& table = record.GetPathDescription().GetColumnTableDescription(); @@ -913,59 +913,59 @@ TCheckFunc HasOlapTableTtlSettingsTiering(ui32 tierNo, const TString& tierName, }; } -void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) { - const auto& self = record.GetPathDescription().GetSelf(); - TSecurityObject src(self.GetOwner(), self.GetEffectiveACL(), false); +void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) { + const auto& self = record.GetPathDescription().GetSelf(); + TSecurityObject src(self.GetOwner(), self.GetEffectiveACL(), false); - NACLib::TSecurityObject required; - required.FromString(right); + NACLib::TSecurityObject required; + required.FromString(right); - for (const auto& requeriedAce : required.GetACL().GetACE()) { - bool has = false; + for (const auto& requeriedAce : required.GetACL().GetACE()) { + bool has = false; - for (const auto& srcAce: src.GetACL().GetACE()) { - if (srcAce.GetAccessType() == requeriedAce.GetAccessType() && - srcAce.GetAccessRight() == requeriedAce.GetAccessRight() && - srcAce.GetSID() == requeriedAce.GetSID() && - srcAce.GetInheritanceType() == requeriedAce.GetInheritanceType()) - { - has = true; + for (const auto& srcAce: src.GetACL().GetACE()) { + if (srcAce.GetAccessType() == requeriedAce.GetAccessType() && + srcAce.GetAccessRight() == requeriedAce.GetAccessRight() && + srcAce.GetSID() == requeriedAce.GetSID() && + srcAce.GetInheritanceType() == requeriedAce.GetInheritanceType()) + { + has = true; } } - UNIT_ASSERT_C(!(has ^ mustHave), "" << (mustHave ? "no " : "") << "ace found" - << ", got " << src.ShortDebugString() - << ", required " << required.ShortDebugString()); - } -} - -TCheckFunc HasEffectiveRight(const TString& right) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - CheckEffectiveRight(record, right, true); + UNIT_ASSERT_C(!(has ^ mustHave), "" << (mustHave ? "no " : "") << "ace found" + << ", got " << src.ShortDebugString() + << ", required " << required.ShortDebugString()); + } +} + +TCheckFunc HasEffectiveRight(const TString& right) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + CheckEffectiveRight(record, right, true); }; } TCheckFunc HasNotEffectiveRight(const TString& right) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - CheckEffectiveRight(record, right, false); + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + CheckEffectiveRight(record, right, false); }; } TCheckFunc KesusConfigIs(ui64 self_check_period_millis, ui64 session_grace_period_millis) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { const auto& config = record.GetPathDescription().GetKesus().GetConfig(); - UNIT_ASSERT_VALUES_EQUAL(config.self_check_period_millis(), self_check_period_millis); - UNIT_ASSERT_VALUES_EQUAL(config.session_grace_period_millis(), session_grace_period_millis); + UNIT_ASSERT_VALUES_EQUAL(config.self_check_period_millis(), self_check_period_millis); + UNIT_ASSERT_VALUES_EQUAL(config.session_grace_period_millis(), session_grace_period_millis); }; } TCheckFunc DatabaseQuotas(ui64 dataStreamShards) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); - + UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); + const auto& pathDescr = record.GetPathDescription(); const auto& domain = pathDescr.GetDomainDescription(); - const auto count = domain.GetDatabaseQuotas().data_stream_shards_quota(); + const auto count = domain.GetDatabaseQuotas().data_stream_shards_quota(); UNIT_ASSERT_C(count == dataStreamShards, "data stream shards inside domain count mistmach, domain with id " << domain.GetDomainKey().GetPathId() << @@ -976,37 +976,37 @@ TCheckFunc DatabaseQuotas(ui64 dataStreamShards) { TCheckFunc PartitionKeys(TVector<TString> lastShardKeys) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& pathDescr = record.GetPathDescription(); - UNIT_ASSERT_VALUES_EQUAL(lastShardKeys.size(), pathDescr.TablePartitionsSize()); + const auto& pathDescr = record.GetPathDescription(); + UNIT_ASSERT_VALUES_EQUAL(lastShardKeys.size(), pathDescr.TablePartitionsSize()); for (size_t i = 0; i < lastShardKeys.size(); ++i) { - UNIT_ASSERT_STRING_CONTAINS(pathDescr.GetTablePartitions(i).GetEndOfRangeKeyPrefix(), lastShardKeys[i]); + UNIT_ASSERT_STRING_CONTAINS(pathDescr.GetTablePartitions(i).GetEndOfRangeKeyPrefix(), lastShardKeys[i]); } }; } -} // NLs -} // NSchemeShardUT_Private - +} // NLs +} // NSchemeShardUT_Private + Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::TTTLSettings::EUnit, o, x) { o << NKikimrSchemeOp::TTTLSettings::EUnit_Name(x); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimrScheme::EStatus, o, x) { - o << NKikimrScheme::EStatus_Name(x); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EPathType, o, x) { - o << NKikimrSchemeOp::EPathType_Name(x); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EFreezeState, o, x) { - o << NKikimrSchemeOp::EFreezeState_Name(x); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EIndexType, o, x) { - o << NKikimrSchemeOp::EIndexType_Name(x); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EIndexState, o, x) { - o << NKikimrSchemeOp::EIndexState_Name(x); -} +} + +Y_DECLARE_OUT_SPEC(inline, NKikimrScheme::EStatus, o, x) { + o << NKikimrScheme::EStatus_Name(x); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EPathType, o, x) { + o << NKikimrSchemeOp::EPathType_Name(x); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EFreezeState, o, x) { + o << NKikimrSchemeOp::EFreezeState_Name(x); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EIndexType, o, x) { + o << NKikimrSchemeOp::EIndexType_Name(x); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimrSchemeOp::EIndexState, o, x) { + o << NKikimrSchemeOp::EIndexState_Name(x); +} diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 9d665d49ee8..33022e42576 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -88,17 +88,17 @@ namespace NLs { TCheckFunc AllowFollowerPromotion(bool val); TCheckFunc FollowerGroups(const TVector<NKikimrHive::TFollowerGroup>& followerGroup = TVector<NKikimrHive::TFollowerGroup>{}); TCheckFunc SizeToSplitEqual(ui32 size); - TCheckFunc MinPartitionsCountEqual(ui32 count); - TCheckFunc MaxPartitionsCountEqual(ui32 count); - TCheckFunc PartitioningByLoadStatus(bool status); + TCheckFunc MinPartitionsCountEqual(ui32 count); + TCheckFunc MaxPartitionsCountEqual(ui32 count); + TCheckFunc PartitioningByLoadStatus(bool status); TCheckFunc ColumnFamiliesCount(ui32 size); TCheckFunc ColumnFamiliesHas(ui32 familyId); TCheckFunc ColumnFamiliesHas(ui32 familyId, const TString& familyName); - TCheckFunc KeyBloomFilterStatus(bool status); - TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter, + TCheckFunc KeyBloomFilterStatus(bool status); + TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter, NKikimrSchemeOp::TTTLSettings::EUnit columnUnit = NKikimrSchemeOp::TTTLSettings::UNIT_AUTO); - TCheckFunc HasTtlDisabled(); - TCheckFunc IsBackupTable(bool value); + TCheckFunc HasTtlDisabled(); + TCheckFunc IsBackupTable(bool value); TCheckFunc HasOlapTableSchemaPreset(const TString& presetName); TCheckFunc HasOlapTableSchemaVersion(ui64 schemaVersion); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 284d88a33b8..9cfd8037b7e 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -337,15 +337,15 @@ private: HFunc(TEvSchemeShard::TEvCancelTx, Handle); HFunc(TEvSchemeShard::TEvCancelTxResult, Handle); - - HFunc(TEvExport::TEvCancelExportRequest, Handle); - HFunc(TEvExport::TEvCancelExportResponse, Handle); - - HFunc(TEvExport::TEvForgetExportRequest, Handle); - HFunc(TEvExport::TEvForgetExportResponse, Handle); - - HFunc(TEvImport::TEvCancelImportRequest, Handle); - HFunc(TEvImport::TEvCancelImportResponse, Handle); + + HFunc(TEvExport::TEvCancelExportRequest, Handle); + HFunc(TEvExport::TEvCancelExportResponse, Handle); + + HFunc(TEvExport::TEvForgetExportRequest, Handle); + HFunc(TEvExport::TEvForgetExportResponse, Handle); + + HFunc(TEvImport::TEvCancelImportRequest, Handle); + HFunc(TEvImport::TEvCancelImportResponse, Handle); }; } @@ -391,18 +391,18 @@ private: HandleRequest(ev, ctx); } - void Handle(TEvExport::TEvCancelExportRequest::TPtr &ev, const TActorContext &ctx) { - HandleRequest(ev, ctx); - } - - void Handle(TEvExport::TEvForgetExportRequest::TPtr &ev, const TActorContext &ctx) { - HandleRequest(ev, ctx); - } - - void Handle(TEvImport::TEvCancelImportRequest::TPtr &ev, const TActorContext &ctx) { - HandleRequest(ev, ctx); - } - + void Handle(TEvExport::TEvCancelExportRequest::TPtr &ev, const TActorContext &ctx) { + HandleRequest(ev, ctx); + } + + void Handle(TEvExport::TEvForgetExportRequest::TPtr &ev, const TActorContext &ctx) { + HandleRequest(ev, ctx); + } + + void Handle(TEvImport::TEvCancelImportRequest::TPtr &ev, const TActorContext &ctx) { + HandleRequest(ev, ctx); + } + template<class TEventPtr> void HandleResponse(TEventPtr &ev, const TActorContext &ctx) { ui64 txId = ev->Get()->Record.GetTxId(); @@ -423,18 +423,18 @@ private: HandleResponse(ev, ctx); } - void Handle(TEvExport::TEvCancelExportResponse::TPtr &ev, const TActorContext &ctx) { - HandleResponse(ev, ctx); - } - - void Handle(TEvExport::TEvForgetExportResponse::TPtr &ev, const TActorContext &ctx) { - HandleResponse(ev, ctx); - } - - void Handle(TEvImport::TEvCancelImportResponse::TPtr &ev, const TActorContext &ctx) { - HandleResponse(ev, ctx); - } - + void Handle(TEvExport::TEvCancelExportResponse::TPtr &ev, const TActorContext &ctx) { + HandleResponse(ev, ctx); + } + + void Handle(TEvExport::TEvForgetExportResponse::TPtr &ev, const TActorContext &ctx) { + HandleResponse(ev, ctx); + } + + void Handle(TEvImport::TEvCancelImportResponse::TPtr &ev, const TActorContext &ctx) { + HandleResponse(ev, ctx); + } + void SendToSchemeshard(ui64 txId, const TActorContext &ctx) { if (!SchemeShardPipe) { SchemeShardPipe = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, SchemeshardTabletId, GetPipeConfigWithRetries())); @@ -485,7 +485,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe : SchemeShardFactory(ssFactory) , HiveState(new TFakeHiveState) , CoordinatorState(new TFakeCoordinator::TState) - , ChannelsCount(opts.NChannels_) + , ChannelsCount(opts.NChannels_) { ui64 hive = TTestTxConfig::Hive; ui64 schemeRoot = TTestTxConfig::SchemeShard; @@ -495,22 +495,22 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe TAppPrepare app(dsExportFactory ? dsExportFactory : static_cast<std::shared_ptr<NKikimr::NDataShard::IExportFactory>>(std::make_shared<TDataShardExportFactory>())); app.SetEnableDataColumnForIndexTable(true); - app.SetEnableSystemViews(opts.EnableSystemViews_); - app.SetEnablePersistentPartitionStats(opts.EnablePersistentPartitionStats_); - app.SetEnableTtlOnAsyncIndexedTables(opts.EnableTtlOnAsyncIndexedTables_); - app.SetAllowUpdateChannelsBindingOfSolomonPartitions(opts.AllowUpdateChannelsBindingOfSolomonPartitions_); - app.SetEnableAsyncIndexes(opts.EnableAsyncIndexes_); + app.SetEnableSystemViews(opts.EnableSystemViews_); + app.SetEnablePersistentPartitionStats(opts.EnablePersistentPartitionStats_); + app.SetEnableTtlOnAsyncIndexedTables(opts.EnableTtlOnAsyncIndexedTables_); + app.SetAllowUpdateChannelsBindingOfSolomonPartitions(opts.AllowUpdateChannelsBindingOfSolomonPartitions_); + app.SetEnableAsyncIndexes(opts.EnableAsyncIndexes_); app.SetEnableNotNullColumns(opts.EnableNotNullColumns_); app.SetEnableSchemeTransactionsAtSchemeShard(opts.EnableSchemeTransactionsAtSchemeShard_); app.SetEnableOlapSchemaOperations(opts.EnableOlapSchemaOperations_); - app.SetEnableProtoSourceIdInfo(opts.EnableProtoSourceIdInfo_); + app.SetEnableProtoSourceIdInfo(opts.EnableProtoSourceIdInfo_); app.SetEnableBackgroundCompaction(opts.EnableBackgroundCompaction_); app.FeatureFlags.SetEnablePublicApiExternalBlobs(true); - for (const auto& sid : opts.SystemBackupSIDs_) { - app.AddSystemBackupSID(sid); - } - + for (const auto& sid : opts.SystemBackupSIDs_) { + app.AddSystemBackupSID(sid); + } + AddDomain(runtime, app, TTestTxConfig::DomainUid, 0, hive, schemeRoot); SetupLogging(runtime); @@ -521,7 +521,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe } SetupTabletServices(runtime, &app); - if (opts.EnablePipeRetries_) { + if (opts.EnablePipeRetries_) { EnableSchemeshardPipeRetriesGuard = EnableSchemeshardPipeRetries(runtime); } @@ -548,15 +548,15 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe SetSplitMergePartCountLimit(&runtime, -1); } -NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime &runtime, ui32 nchannels, bool enablePipeRetries, - NSchemeShardUT_Private::TTestEnv::TSchemeShardFactory ssFactory, bool enableSystemViews) - : TTestEnv(runtime, TTestEnvOptions() - .NChannels(nchannels) - .EnablePipeRetries(enablePipeRetries) - .EnableSystemViews(enableSystemViews), ssFactory) -{ -} - +NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime &runtime, ui32 nchannels, bool enablePipeRetries, + NSchemeShardUT_Private::TTestEnv::TSchemeShardFactory ssFactory, bool enableSystemViews) + : TTestEnv(runtime, TTestEnvOptions() + .NChannels(nchannels) + .EnablePipeRetries(enablePipeRetries) + .EnableSystemViews(enableSystemViews), ssFactory) +{ +} + void NSchemeShardUT_Private::TTestEnv::SetupLogging(TTestActorRuntime &runtime) { runtime.SetLogPriority(NKikimrServices::PERSQUEUE, NActors::NLog::PRI_ERROR); runtime.SetLogPriority(NKikimrServices::BS_CONTROLLER, NActors::NLog::PRI_ERROR); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index f03200d4c65..72b934c570d 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -20,33 +20,33 @@ namespace NSchemeShardUT_Private { NActors::TActorId CreateNotificationSubscriber(NActors::TTestActorRuntime &runtime, ui64 schemeshardId); NActors::TActorId CreateFakeMetering(NActors::TTestActorRuntime &runtime); - struct TTestEnvOptions { - using TSelf = TTestEnvOptions; - - #define OPTION(type, name, defaultValue) \ - TSelf& name(type value) { \ - name##_ = value; \ - return *this; \ - } \ - type name##_ = defaultValue - - OPTION(ui32, NChannels, 4); - OPTION(bool, EnablePipeRetries, true); - OPTION(std::optional<bool>, EnableSystemViews, std::nullopt); - OPTION(std::optional<bool>, EnablePersistentPartitionStats, std::nullopt); - OPTION(std::optional<bool>, EnableTtlOnAsyncIndexedTables, std::nullopt); - OPTION(std::optional<bool>, AllowUpdateChannelsBindingOfSolomonPartitions, std::nullopt); - OPTION(std::optional<bool>, EnableAsyncIndexes, std::nullopt); - OPTION(std::optional<bool>, EnableNotNullColumns, std::nullopt); + struct TTestEnvOptions { + using TSelf = TTestEnvOptions; + + #define OPTION(type, name, defaultValue) \ + TSelf& name(type value) { \ + name##_ = value; \ + return *this; \ + } \ + type name##_ = defaultValue + + OPTION(ui32, NChannels, 4); + OPTION(bool, EnablePipeRetries, true); + OPTION(std::optional<bool>, EnableSystemViews, std::nullopt); + OPTION(std::optional<bool>, EnablePersistentPartitionStats, std::nullopt); + OPTION(std::optional<bool>, EnableTtlOnAsyncIndexedTables, std::nullopt); + OPTION(std::optional<bool>, AllowUpdateChannelsBindingOfSolomonPartitions, std::nullopt); + OPTION(std::optional<bool>, EnableAsyncIndexes, std::nullopt); + OPTION(std::optional<bool>, EnableNotNullColumns, std::nullopt); OPTION(std::optional<bool>, EnableSchemeTransactionsAtSchemeShard, std::nullopt); OPTION(std::optional<bool>, EnableOlapSchemaOperations, std::nullopt); - OPTION(std::optional<bool>, EnableProtoSourceIdInfo, std::nullopt); + OPTION(std::optional<bool>, EnableProtoSourceIdInfo, std::nullopt); OPTION(std::optional<bool>, EnableBackgroundCompaction, std::nullopt); - OPTION(THashSet<TString>, SystemBackupSIDs, {}); - - #undef OPTION - }; - + OPTION(THashSet<TString>, SystemBackupSIDs, {}); + + #undef OPTION + }; + class TTestEnv { public: using TSchemeShardFactory = std::function<IActor* (const TActorId &, TTabletStorageInfo *)>; @@ -64,16 +64,16 @@ namespace NSchemeShardUT_Private { public: TTestEnv(TTestActorRuntime& runtime, ui32 nchannels = 4, bool enablePipeRetries = true, TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, bool enableSystemViews = false); - TTestEnv(TTestActorRuntime& runtime, const TTestEnvOptions& opts, + TTestEnv(TTestActorRuntime& runtime, const TTestEnvOptions& opts, TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, std::shared_ptr<NKikimr::NDataShard::IExportFactory> dsExportFactory = {}); TFakeHiveState::TPtr GetHiveState() const; TAutoPtr<ITabletScheduledEventsGuard> EnableSchemeshardPipeRetries(TTestActorRuntime& runtime); ui32 ReliablePropose(TTestActorRuntime& runtime, TEvSchemeShard::TEvModifySchemeTransaction* evTx, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); ui32 ReliablePropose(TTestActorRuntime& runtime, TEvSchemeShard::TEvCancelTx* evTx, const TVector<TEvSchemeShard::EStatus>& expectedResults = {NKikimrScheme::StatusAccepted}); - ui32 ReliablePropose(TTestActorRuntime& runtime, TEvExport::TEvCancelExportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); - ui32 ReliablePropose(TTestActorRuntime& runtime, TEvExport::TEvForgetExportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); - ui32 ReliablePropose(TTestActorRuntime& runtime, TEvImport::TEvCancelImportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); + ui32 ReliablePropose(TTestActorRuntime& runtime, TEvExport::TEvCancelExportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); + ui32 ReliablePropose(TTestActorRuntime& runtime, TEvExport::TEvForgetExportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); + ui32 ReliablePropose(TTestActorRuntime& runtime, TEvImport::TEvCancelImportRequest* ev, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses = {Ydb::StatusIds::SUCCESS}); template <class TContainer> void TestWaitNotification(TTestActorRuntime& runtime, TContainer txs, ui64 schemeshardId = TTestTxConfig::SchemeShard) { TSet<ui64> set(txs.begin(), txs.end()); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ya.make b/ydb/core/tx/schemeshard/ut_helpers/ya.make index 8d496331383..b66d799ec0c 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ya.make +++ b/ydb/core/tx/schemeshard/ut_helpers/ya.make @@ -1,8 +1,8 @@ -LIBRARY() - -OWNER(g:kikimr) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +PEERDIR( library/cpp/testing/unittest ydb/core/base ydb/core/blockstore/core @@ -20,10 +20,10 @@ PEERDIR( ydb/core/tx/tx_allocator ydb/core/tx/tx_proxy ydb/public/lib/scheme_types -) - -SRCS( - export_reboots_common.cpp +) + +SRCS( + export_reboots_common.cpp failing_mtpq.cpp test_env.cpp test_env.h @@ -31,8 +31,8 @@ SRCS( ls_checks.h helpers.cpp helpers.h -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/tx/schemeshard/ut_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build.cpp index 23a6868e5ca..88860481cc5 100644 --- a/ydb/core/tx/schemeshard/ut_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build.cpp @@ -195,18 +195,18 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { "Name: \"ResourceDB\""); env.TestWaitNotification(runtime, txId); - const auto attrs = AlterUserAttrs({ - {"cloud_id", "CLOUD_ID_VAL"}, - {"folder_id", "FOLDER_ID_VAL"}, - {"database_id", "DATABASE_ID_VAL"} - }); - - TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "ServerLessDB" - ResourcesDomainKey { - SchemeShard: %lu - PathId: 2 - } + const auto attrs = AlterUserAttrs({ + {"cloud_id", "CLOUD_ID_VAL"}, + {"folder_id", "FOLDER_ID_VAL"}, + {"database_id", "DATABASE_ID_VAL"} + }); + + TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "ServerLessDB" + ResourcesDomainKey { + SchemeShard: %lu + PathId: 2 + } )", TTestTxConfig::SchemeShard), attrs); env.TestWaitNotification(runtime, txId); @@ -260,7 +260,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { } runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE); TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB/Table"), {NLs::PathExist, @@ -291,9 +291,9 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { auto descr = TestGetBuilIndex(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB", txId); Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); - const TString meteringData = R"({"usage":{"start":0,"quantity":179,"finish":0,"unit":"request_unit","type":"delta"},"tags":{},"id":"106-9437199-2-101-1818-101-1818","cloud_id":"CLOUD_ID_VAL","source_wt":0,"source_id":"sless-docapi-ydb-ss","resource_id":"DATABASE_ID_VAL","schema":"ydb.serverless.requests.v1","folder_id":"FOLDER_ID_VAL","version":"1.0.0"})"; - - UNIT_ASSERT_NO_DIFF(meteringMessages, meteringData + "\n"); + const TString meteringData = R"({"usage":{"start":0,"quantity":179,"finish":0,"unit":"request_unit","type":"delta"},"tags":{},"id":"106-9437199-2-101-1818-101-1818","cloud_id":"CLOUD_ID_VAL","source_wt":0,"source_id":"sless-docapi-ydb-ss","resource_id":"DATABASE_ID_VAL","schema":"ydb.serverless.requests.v1","folder_id":"FOLDER_ID_VAL","version":"1.0.0"})"; + + UNIT_ASSERT_NO_DIFF(meteringMessages, meteringData + "\n"); TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB/Table"), {NLs::PathExist, @@ -308,10 +308,10 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { listing = TestListBuilIndex(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB"); Y_ASSERT(listing.EntriesSize() == 0); - TestDropTableIndex(runtime, tenantSchemeShard, ++txId, "/MyRoot/ServerLessDB", R"( - TableName: "Table" - IndexName: "index1" - )"); + TestDropTableIndex(runtime, tenantSchemeShard, ++txId, "/MyRoot/ServerLessDB", R"( + TableName: "Table" + IndexName: "index1" + )"); env.TestWaitNotification(runtime, txId, tenantSchemeShard); RebootTablet(runtime, tenantSchemeShard, runtime.AllocateEdgeActor()); @@ -337,65 +337,65 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { TestBuilIndex(runtime, ++txId, tenantSchemeShard, "/MyRoot/ServerLessDB", "/MyRoot/ServerLessDB/Table", "index2", {"index"}); env.TestWaitNotification(runtime, txId, tenantSchemeShard); - - // CommonDB - TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", - "Name: \"CommonDB\""); - env.TestWaitNotification(runtime, txId); - - TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", - "StoragePools { " - " Name: \"pool-3\" " - " Kind: \"pool-kind-3\" " - "} " - "PlanResolution: 50 " - "Coordinators: 1 " - "Mediators: 1 " - "TimeCastBucketsPerMediator: 2 " - "ExternalSchemeShard: true " - "Name: \"CommonDB\""); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/CommonDB"), - {NLs::PathExist, - NLs::IsExternalSubDomain("CommonDB"), - NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - - TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/CommonDB", R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "index" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId, tenantSchemeShard); - - for (ui32 delta = 0; delta < 101; ++delta) { + + // CommonDB + TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", + "Name: \"CommonDB\""); + env.TestWaitNotification(runtime, txId); + + TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", + "StoragePools { " + " Name: \"pool-3\" " + " Kind: \"pool-kind-3\" " + "} " + "PlanResolution: 50 " + "Coordinators: 1 " + "Mediators: 1 " + "TimeCastBucketsPerMediator: 2 " + "ExternalSchemeShard: true " + "Name: \"CommonDB\""); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/CommonDB"), + {NLs::PathExist, + NLs::IsExternalSubDomain("CommonDB"), + NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); + + TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/CommonDB", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "index" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId, tenantSchemeShard); + + for (ui32 delta = 0; delta < 101; ++delta) { fnWriteRow(TTestTxConfig::FakeHiveTablets + 12, 1 + delta, 1000 + delta, "aaaa", "Table"); - } - - TVector<TString> billRecords; - runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { - if (ev->Type == NMetering::TEvMetering::TEvWriteMeteringJson::EventType) { - auto* msg = ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>(); - billRecords.push_back(msg->MeteringJson); - } - - return TTestActorRuntime::EEventAction::PROCESS; - }); - - TestBuilIndex(runtime, ++txId, tenantSchemeShard, "/MyRoot/CommonDB", "/MyRoot/CommonDB/Table", "index1", {"index"}); - builIndexId = txId; - - listing = TestListBuilIndex(runtime, tenantSchemeShard, "/MyRoot/CommonDB"); - Y_ASSERT(listing.EntriesSize() == 1); - - env.TestWaitNotification(runtime, txId, tenantSchemeShard); - - descr = TestGetBuilIndex(runtime, tenantSchemeShard, "/MyRoot/CommonDB", txId); - Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); - - UNIT_ASSERT(billRecords.empty()); + } + + TVector<TString> billRecords; + runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { + if (ev->Type == NMetering::TEvMetering::TEvWriteMeteringJson::EventType) { + auto* msg = ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>(); + billRecords.push_back(msg->MeteringJson); + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + TestBuilIndex(runtime, ++txId, tenantSchemeShard, "/MyRoot/CommonDB", "/MyRoot/CommonDB/Table", "index1", {"index"}); + builIndexId = txId; + + listing = TestListBuilIndex(runtime, tenantSchemeShard, "/MyRoot/CommonDB"); + Y_ASSERT(listing.EntriesSize() == 1); + + env.TestWaitNotification(runtime, txId, tenantSchemeShard); + + descr = TestGetBuilIndex(runtime, tenantSchemeShard, "/MyRoot/CommonDB", txId); + Y_ASSERT(descr.GetIndexBuild().GetState() == Ydb::Table::IndexBuildState::STATE_DONE); + + UNIT_ASSERT(billRecords.empty()); } Y_UNIT_TEST(CancellationNotEnoughRetries) { @@ -571,10 +571,10 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { auto listing = TestListBuilIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot"); Y_ASSERT(listing.EntriesSize() == 0); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "WithFollowers" - IndexName: "UserDefinedIndexByValue0" - )"); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "WithFollowers" + IndexName: "UserDefinedIndexByValue0" + )"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/WithFollowers"), @@ -681,10 +681,10 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { {NKikimrScheme::StatusMultipleModifications}); env.TestWaitNotification(runtime, txId); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - IndexName: "nameOK" - )", {NKikimrScheme::StatusMultipleModifications}); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "nameOK" + )", {NKikimrScheme::StatusMultipleModifications}); env.TestWaitNotification(runtime, txId); TestDropTable(runtime, ++txId, "/MyRoot", "Table", @@ -746,28 +746,28 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(2)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished, NLs::PathVersionEqual(3)}); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )"); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )"); env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::Finished, NLs::PathVersionEqual(5), NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::PathNotExist}); TestCopyTable(runtime, ++txId, "/MyRoot", "Copy", "/MyRoot/Table"); @@ -777,9 +777,9 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { {NLs::Finished, NLs::PathVersionEqual(5), NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue1"), {NLs::PathExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue1/indexImplTable"), {NLs::PathExist}); @@ -814,50 +814,50 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished, NLs::PathVersionEqual(3)}); - TestDropTableIndex(runtime, ++txId, "/MyRoot/NotExist", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )", {NKikimrScheme::StatusPathDoesNotExist}); + TestDropTableIndex(runtime, ++txId, "/MyRoot/NotExist", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "NotExist" - IndexName: "UserDefinedIndexByValue0" - )", {NKikimrScheme::StatusPathDoesNotExist}); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "NotExist" + IndexName: "UserDefinedIndexByValue0" + )", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - IndexName: "NotExist" - )", {NKikimrScheme::StatusPathDoesNotExist}); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "NotExist" + )", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )"); - TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )", {NKikimrScheme::StatusMultipleModifications}); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )"); + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )", {NKikimrScheme::StatusMultipleModifications}); env.TestWaitNotification(runtime, {txId, txId - 1}); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::Finished, NLs::PathVersionEqual(5), NLs::IndexesCount(0)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::PathNotExist}); TestDropTable(runtime, ++txId, "/MyRoot", "Table"); diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp index 1898472b344..d270fd87e67 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp @@ -230,29 +230,29 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished, NLs::PathVersionEqual(3)}); } - TestDropTableIndex(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )"); + TestDropTableIndex(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )"); t.TestEnv->TestWaitNotification(runtime, t.TxId); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::Finished, NLs::PathVersionEqual(5), NLs::IndexesCount(0)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::PathNotExist}); }); } @@ -283,29 +283,29 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { {NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished, NLs::PathVersionEqual(3)}); } - TestDropTableIndex(runtime, ++t.TxId, "/MyRoot", R"( - TableName: "Table" - IndexName: "UserDefinedIndexByValue0" - )"); + TestDropTableIndex(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + IndexName: "UserDefinedIndexByValue0" + )"); t.TestEnv->TestWaitNotification(runtime, t.TxId); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::Finished, NLs::PathVersionEqual(5), NLs::IndexesCount(0)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0"), {NLs::PathNotExist}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndexByValue0/indexImplTable"), {NLs::PathNotExist}); }); } diff --git a/ydb/core/tx/schemeshard/ut_move.cpp b/ydb/core/tx/schemeshard/ut_move.cpp index 62d5e90473c..bb20b599583 100644 --- a/ydb/core/tx/schemeshard/ut_move.cpp +++ b/ydb/core/tx/schemeshard/ut_move.cpp @@ -1,6 +1,6 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard_utils.h> - + #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/tx/datashard/change_exchange.h> diff --git a/ydb/core/tx/schemeshard/ut_move_reboots.cpp b/ydb/core/tx/schemeshard/ut_move_reboots.cpp index daee5ff52f3..58e9bebd9ba 100644 --- a/ydb/core/tx/schemeshard/ut_move_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_move_reboots.cpp @@ -1,6 +1,6 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard_utils.h> - + #include <ydb/core/base/compile_time_flags.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/tx/datashard/change_exchange.h> diff --git a/ydb/core/tx/schemeshard/ut_olap_reboots.cpp b/ydb/core/tx/schemeshard/ut_olap_reboots.cpp index 8e4a9d639ad..c7369d4a4e4 100644 --- a/ydb/core/tx/schemeshard/ut_olap_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_olap_reboots.cpp @@ -9,7 +9,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(CreateStore) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -45,7 +45,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(CreateTable) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -84,7 +84,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(CreateDropTable) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -125,7 +125,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(CreateMultipleTables) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -170,7 +170,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(DropMultipleTables) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -221,7 +221,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(CreateDropStore) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -263,7 +263,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(DropTableThenStore) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); @@ -311,7 +311,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) { Y_UNIT_TEST(AlterTtlSettings) { TTestWithReboots t(false); t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); + runtime.GetAppData().FeatureFlags.SetEnableOlapSchemaOperationsForTest(true); { TInactiveZone inactive(activeZone); diff --git a/ydb/core/tx/schemeshard/ut_reboots.cpp b/ydb/core/tx/schemeshard/ut_reboots.cpp index e93771ed07e..dce2c127122 100644 --- a/ydb/core/tx/schemeshard/ut_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_reboots.cpp @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(IntermediateDirsReboots) { TTestWithReboots t; t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { AsyncMkDir(runtime, ++t.TxId, "/MyRoot", "x/y/z"); - TestForceDropUnsafe(runtime, ++t.TxId, 4, TVector<NKikimrScheme::EStatus>{NKikimrScheme::StatusMultipleModifications}); + TestForceDropUnsafe(runtime, ++t.TxId, 4, TVector<NKikimrScheme::EStatus>{NKikimrScheme::StatusMultipleModifications}); t.TestEnv->TestWaitNotification(runtime, {t.TxId - 1, t.TxId}); { @@ -219,7 +219,7 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { {NLs::PathVersionEqual(7)}); } - t.TestEnv->ReliablePropose(runtime, ConsistentCopyTablesRequest(++t.TxId, "/", R"( + t.TestEnv->ReliablePropose(runtime, ConsistentCopyTablesRequest(++t.TxId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirB/src1" DstPath: "/MyRoot/DirB/dst1" @@ -296,7 +296,7 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { }; fnWriteRow(TTestTxConfig::FakeHiveTablets); - TestConsistentCopyTables(runtime, ++t.TxId, "/", R"( + TestConsistentCopyTables(runtime, ++t.TxId, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/DirB/src1" DstPath: "/MyRoot/DirB/dst1" @@ -373,27 +373,27 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { { NLs::Finished, NLs::PathVersionEqual(3), NLs::IndexesCount(3)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValues"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValues"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateNotReady), NLs::IndexKeys({"value0", "value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValues/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValues/indexImplTable"), {NLs::Finished}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1"), {NLs::Finished, NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobal), NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), NLs::IndexKeys({"value0"}), NLs::IndexDataColumns({"value1"})}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DirB/Table1/UserDefinedIndexByValue0CoveringValue1/indexImplTable"), {NLs::Finished}); } diff --git a/ydb/core/tx/schemeshard/ut_replication.cpp b/ydb/core/tx/schemeshard/ut_replication.cpp index 46efb783b53..0f511f2188d 100644 --- a/ydb/core/tx/schemeshard/ut_replication.cpp +++ b/ydb/core/tx/schemeshard/ut_replication.cpp @@ -1,96 +1,96 @@ -#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(TReplicationTests) { - void SetupLogging(TTestActorRuntimeBase& runtime) { - runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::REPLICATION_CONTROLLER, NActors::NLog::PRI_TRACE); - } - - Y_UNIT_TEST(Create) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - SetupLogging(runtime); - - TestCreateReplication(runtime, ++txId, "/MyRoot", R"( - Name: "Replication" - )"); - env.TestWaitNotification(runtime, txId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - } - - Y_UNIT_TEST(CreateSequential) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - SetupLogging(runtime); - - for (int i = 0; i < 2; ++i) { - const auto name = Sprintf("Replication%d", i); - - TestCreateReplication(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "%s" - )", name.c_str())); - env.TestWaitNotification(runtime, txId); - TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); - } - } - - Y_UNIT_TEST(CreateInParallel) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - SetupLogging(runtime); - - for (int i = 0; i < 2; ++i) { - TVector<TString> names; - TVector<ui64> txIds; - - for (int j = 0; j < 2; ++j) { - auto name = Sprintf("Replication%d-%d", i, j); - - TestCreateReplication(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "%s" - )", name.c_str())); - - names.push_back(std::move(name)); - txIds.push_back(txId); - } - - env.TestWaitNotification(runtime, txIds); - for (const auto& name : names) { - TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); - } - } - } - - Y_UNIT_TEST(CreateDropRecreate) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - SetupLogging(runtime); - - TestCreateReplication(runtime, ++txId, "/MyRoot", R"( - Name: "Replication" - )"); - env.TestWaitNotification(runtime, txId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - - TestDropReplication(runtime, ++txId, "/MyRoot", "Replication"); - env.TestWaitNotification(runtime, txId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathNotExist); - - TestCreateReplication(runtime, ++txId, "/MyRoot", R"( - Name: "Replication" - )"); - env.TestWaitNotification(runtime, txId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - } - -} // TReplicationTests +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TReplicationTests) { + void SetupLogging(TTestActorRuntimeBase& runtime) { + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::REPLICATION_CONTROLLER, NActors::NLog::PRI_TRACE); + } + + Y_UNIT_TEST(Create) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + TestCreateReplication(runtime, ++txId, "/MyRoot", R"( + Name: "Replication" + )"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + } + + Y_UNIT_TEST(CreateSequential) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + for (int i = 0; i < 2; ++i) { + const auto name = Sprintf("Replication%d", i); + + TestCreateReplication(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "%s" + )", name.c_str())); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); + } + } + + Y_UNIT_TEST(CreateInParallel) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + for (int i = 0; i < 2; ++i) { + TVector<TString> names; + TVector<ui64> txIds; + + for (int j = 0; j < 2; ++j) { + auto name = Sprintf("Replication%d-%d", i, j); + + TestCreateReplication(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "%s" + )", name.c_str())); + + names.push_back(std::move(name)); + txIds.push_back(txId); + } + + env.TestWaitNotification(runtime, txIds); + for (const auto& name : names) { + TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); + } + } + } + + Y_UNIT_TEST(CreateDropRecreate) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + TestCreateReplication(runtime, ++txId, "/MyRoot", R"( + Name: "Replication" + )"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + + TestDropReplication(runtime, ++txId, "/MyRoot", "Replication"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathNotExist); + + TestCreateReplication(runtime, ++txId, "/MyRoot", R"( + Name: "Replication" + )"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + } + +} // TReplicationTests diff --git a/ydb/core/tx/schemeshard/ut_replication/ya.make b/ydb/core/tx/schemeshard/ut_replication/ya.make index ebb41d1d7f0..a9c36ad0842 100644 --- a/ydb/core/tx/schemeshard/ut_replication/ya.make +++ b/ydb/core/tx/schemeshard/ut_replication/ya.make @@ -1,31 +1,31 @@ -UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(2) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( - ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_replication.cpp -) - -YQL_LAST_ABI_VERSION() - -END() +UNITTEST_FOR(ydb/core/tx/schemeshard) + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(2) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/tx/schemeshard/ut_helpers +) + +SRCS( + ut_replication.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ut_replication_reboots.cpp b/ydb/core/tx/schemeshard/ut_replication_reboots.cpp index ebb559a5714..45d4e97640a 100644 --- a/ydb/core/tx/schemeshard/ut_replication_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_replication_reboots.cpp @@ -1,129 +1,129 @@ -#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - -using namespace NSchemeShardUT_Private; - -Y_UNIT_TEST_SUITE(TReplicationWithRebootsTests) { - void SetupLogging(TTestActorRuntimeBase& runtime) { - runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::REPLICATION_CONTROLLER, NActors::NLog::PRI_TRACE); - } - - Y_UNIT_TEST(Create) { - TTestWithReboots t(false); - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - SetupLogging(runtime); - } - - TestCreateReplication(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Replication" - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - }); - } - - void CreateMultipleReplications(bool withInitialController) { - TTestWithReboots t(false); - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - SetupLogging(runtime); - - if (withInitialController) { - TestCreateReplication(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Replication0" - )"); - - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestLs(runtime, "/MyRoot/Replication0", false, NLs::PathExist); - } - } - - TVector<TString> names; - TVector<ui64> txIds; - - for (int i = 1; i <= 3; ++i) { - auto name = Sprintf("Replication%d", i); - auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", Sprintf(R"( - Name: "%s" - )", name.c_str())); - - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusAlreadyExists, - NKikimrScheme::StatusMultipleModifications, - }); - - names.push_back(std::move(name)); - txIds.push_back(t.TxId); - } - t.TestEnv->TestWaitNotification(runtime, txIds); - - { - TInactiveZone inactive(activeZone); - - for (const auto& name : names) { - TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); - } - } - }); - } - - Y_UNIT_TEST(CreateInParallelWithoutInitialController) { - CreateMultipleReplications(false); - } - - Y_UNIT_TEST(CreateInParallelWithInitialController) { - CreateMultipleReplications(true); - } - - Y_UNIT_TEST(CreateDropRecreate) { - TTestWithReboots t(false); - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - SetupLogging(runtime); - } - - { - auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", R"( - Name: "Replication" - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusAlreadyExists, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - - { - auto request = DropReplicationRequest(++t.TxId, "/MyRoot", "Replication"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathNotExist); - - { - auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", R"( - Name: "Replication" - )"); - t.TestEnv->ReliablePropose(runtime, request, { - NKikimrScheme::StatusAccepted, - NKikimrScheme::StatusAlreadyExists, - NKikimrScheme::StatusMultipleModifications, - }); - } - t.TestEnv->TestWaitNotification(runtime, t.TxId); - TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); - }); - } - -} // TReplicationWithRebootsTests +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TReplicationWithRebootsTests) { + void SetupLogging(TTestActorRuntimeBase& runtime) { + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::REPLICATION_CONTROLLER, NActors::NLog::PRI_TRACE); + } + + Y_UNIT_TEST(Create) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + SetupLogging(runtime); + } + + TestCreateReplication(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Replication" + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + }); + } + + void CreateMultipleReplications(bool withInitialController) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + SetupLogging(runtime); + + if (withInitialController) { + TestCreateReplication(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Replication0" + )"); + + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestLs(runtime, "/MyRoot/Replication0", false, NLs::PathExist); + } + } + + TVector<TString> names; + TVector<ui64> txIds; + + for (int i = 1; i <= 3; ++i) { + auto name = Sprintf("Replication%d", i); + auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", Sprintf(R"( + Name: "%s" + )", name.c_str())); + + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + + names.push_back(std::move(name)); + txIds.push_back(t.TxId); + } + t.TestEnv->TestWaitNotification(runtime, txIds); + + { + TInactiveZone inactive(activeZone); + + for (const auto& name : names) { + TestLs(runtime, "/MyRoot/" + name, false, NLs::PathExist); + } + } + }); + } + + Y_UNIT_TEST(CreateInParallelWithoutInitialController) { + CreateMultipleReplications(false); + } + + Y_UNIT_TEST(CreateInParallelWithInitialController) { + CreateMultipleReplications(true); + } + + Y_UNIT_TEST(CreateDropRecreate) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + SetupLogging(runtime); + } + + { + auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", R"( + Name: "Replication" + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + + { + auto request = DropReplicationRequest(++t.TxId, "/MyRoot", "Replication"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathNotExist); + + { + auto request = CreateReplicationRequest(++t.TxId, "/MyRoot", R"( + Name: "Replication" + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + } + t.TestEnv->TestWaitNotification(runtime, t.TxId); + TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); + }); + } + +} // TReplicationWithRebootsTests diff --git a/ydb/core/tx/schemeshard/ut_replication_reboots/ya.make b/ydb/core/tx/schemeshard/ut_replication_reboots/ya.make index d46676b0023..3f1e0b3431f 100644 --- a/ydb/core/tx/schemeshard/ut_replication_reboots/ya.make +++ b/ydb/core/tx/schemeshard/ut_replication_reboots/ya.make @@ -1,31 +1,31 @@ -UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(2) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( - ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_replication_reboots.cpp -) - -YQL_LAST_ABI_VERSION() - -END() +UNITTEST_FOR(ydb/core/tx/schemeshard) + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(2) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/tx/schemeshard/ut_helpers +) + +SRCS( + ut_replication_reboots.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore.cpp index 6402ce0341d..a8b33dcbdce 100644 --- a/ydb/core/tx/schemeshard/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore.cpp @@ -1,5 +1,5 @@ -#include <contrib/libs/double-conversion/ieee.h> - +#include <contrib/libs/double-conversion/ieee.h> + #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/tablet/resource_broker.h> #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> @@ -9,1178 +9,1178 @@ #include <ydb/core/wrappers/ut_helpers/s3_mock.h> #include <ydb/core/metering/metering.h> #include <ydb/core/ydb_convert/table_description.h> -#include <library/cpp/string_utils/quote/quote.h> - -#include <util/datetime/base.h> -#include <util/generic/size_literals.h> -#include <util/generic/vector.h> -#include <util/string/builder.h> -#include <util/string/join.h> -#include <util/string/printf.h> - -using namespace NKikimr; -using namespace NSchemeShardUT_Private; +#include <library/cpp/string_utils/quote/quote.h> + +#include <util/datetime/base.h> +#include <util/generic/size_literals.h> +#include <util/generic/vector.h> +#include <util/string/builder.h> +#include <util/string/join.h> +#include <util/string/printf.h> + +using namespace NKikimr; +using namespace NSchemeShardUT_Private; using namespace NKikimr::NSchemeShard; -using namespace NKikimr::NWrappers::NTestHelpers; - -namespace { - - const TString EmptyYsonStr = R"([[[[];%false]]])"; - +using namespace NKikimr::NWrappers::NTestHelpers; + +namespace { + + const TString EmptyYsonStr = R"([[[[];%false]]])"; + TString GenerateScheme(const NKikimrSchemeOp::TPathDescription& pathDesc) { - UNIT_ASSERT(pathDesc.HasTable()); - const auto& tableDesc = pathDesc.GetTable(); - - Ydb::Table::CreateTableRequest scheme; - NKikimrMiniKQL::TType mkqlKeyType; - - scheme.mutable_primary_key()->CopyFrom(tableDesc.GetKeyColumnNames()); - FillColumnDescription(scheme, mkqlKeyType, tableDesc); - FillIndexDescription(scheme, tableDesc); - FillStorageSettings(scheme, tableDesc); - FillColumnFamilies(scheme, tableDesc); - FillAttributes(scheme, pathDesc); - FillTableBoundary(scheme, tableDesc, mkqlKeyType); - FillPartitioningSettings(scheme, tableDesc); - FillKeyBloomFilter(scheme, tableDesc); - FillReadReplicasSettings(scheme, tableDesc); - - TString result; - UNIT_ASSERT(google::protobuf::TextFormat::PrintToString(scheme, &result)); - - return result; - } - + UNIT_ASSERT(pathDesc.HasTable()); + const auto& tableDesc = pathDesc.GetTable(); + + Ydb::Table::CreateTableRequest scheme; + NKikimrMiniKQL::TType mkqlKeyType; + + scheme.mutable_primary_key()->CopyFrom(tableDesc.GetKeyColumnNames()); + FillColumnDescription(scheme, mkqlKeyType, tableDesc); + FillIndexDescription(scheme, tableDesc); + FillStorageSettings(scheme, tableDesc); + FillColumnFamilies(scheme, tableDesc); + FillAttributes(scheme, pathDesc); + FillTableBoundary(scheme, tableDesc, mkqlKeyType); + FillPartitioningSettings(scheme, tableDesc); + FillKeyBloomFilter(scheme, tableDesc); + FillReadReplicasSettings(scheme, tableDesc); + + TString result; + UNIT_ASSERT(google::protobuf::TextFormat::PrintToString(scheme, &result)); + + return result; + } + TString GenerateScheme(const NKikimrScheme::TEvDescribeSchemeResult& describeResult) { - UNIT_ASSERT(describeResult.HasPathDescription()); - return GenerateScheme(describeResult.GetPathDescription()); - } - + UNIT_ASSERT(describeResult.HasPathDescription()); + return GenerateScheme(describeResult.GetPathDescription()); + } + TString GenerateTableDescription(const NKikimrScheme::TEvDescribeSchemeResult& describeResult) { - UNIT_ASSERT(describeResult.HasPathDescription()); - UNIT_ASSERT(describeResult.GetPathDescription().HasTable()); - const auto& tableDesc = describeResult.GetPathDescription().GetTable(); - + UNIT_ASSERT(describeResult.HasPathDescription()); + UNIT_ASSERT(describeResult.GetPathDescription().HasTable()); + const auto& tableDesc = describeResult.GetPathDescription().GetTable(); + NKikimrSchemeOp::TTableDescription scheme; - scheme.MutableColumns()->CopyFrom(tableDesc.GetColumns()); - scheme.MutableKeyColumnNames()->CopyFrom(tableDesc.GetKeyColumnNames()); - - TString schemeStr; - UNIT_ASSERT(google::protobuf::TextFormat::PrintToString(scheme, &schemeStr)); - - return schemeStr; - } - - struct TTestData { - TString Csv; - TString YsonStr; - - TTestData(TString csv, TString ysonStr) - : Csv(std::move(csv)) - , YsonStr(std::move(ysonStr)) - { - } - }; - - struct TTestDataWithScheme { - TString Scheme; - TVector<TTestData> Data; - - TTestDataWithScheme() = default; - - TTestDataWithScheme(TString&& scheme, TVector<TTestData>&& data) - : Scheme(std::move(scheme)) - , Data(std::move(data)) - { - } - }; - - TTestData GenerateTestData(const TString& keyPrefix, ui32 count) { - TStringBuilder csv; - TStringBuilder yson; - - for (ui32 i = 1; i <= count; ++i) { - // csv - if (keyPrefix) { - csv << "\"" << keyPrefix << i << "\","; - } else { - csv << i << ","; - } - - csv << "\"" << "value" << i << "\"" << Endl; - - // yson - if (i == 1) { - yson << "[[[["; - } else { - yson << ";"; - } - - yson << "[" - << "[\"" << keyPrefix << i << "\"];" - << "[\"" << "value" << i << "\"]" - << "]"; - - if (i == count) { - yson << "];\%false]]]"; - } - } - - return TTestData(std::move(csv), std::move(yson)); - } - - TTestDataWithScheme GenerateTestData(const TString& scheme, const TVector<std::pair<TString, ui64>>& shardsConfig) { - TTestDataWithScheme result; - result.Scheme = scheme; - - for (const auto& [keyPrefix, count] : shardsConfig) { - result.Data.push_back(GenerateTestData(keyPrefix, count)); - } - - return result; - } - - THashMap<TString, TString> ConvertTestData(const THashMap<TString, TTestDataWithScheme>& data) { - THashMap<TString, TString> result; - - for (const auto& [prefix, item] : data) { - result.emplace(prefix + "/scheme.pb", item.Scheme); - for (ui32 i = 0; i < item.Data.size(); ++i) { - result.emplace(Sprintf("%s/data_%02d.csv", prefix.data(), i), item.Data.at(i).Csv); - } - } - - return result; - } - - THashMap<TString, TString> ConvertTestData(const TTestDataWithScheme& data) { - return ConvertTestData({{"", data}}); - } - - NKikimrMiniKQL::TResult ReadTableImpl(TTestActorRuntime& runtime, ui64 tabletId, const TString& query) { - NKikimrMiniKQL::TResult result; - - TString error; + scheme.MutableColumns()->CopyFrom(tableDesc.GetColumns()); + scheme.MutableKeyColumnNames()->CopyFrom(tableDesc.GetKeyColumnNames()); + + TString schemeStr; + UNIT_ASSERT(google::protobuf::TextFormat::PrintToString(scheme, &schemeStr)); + + return schemeStr; + } + + struct TTestData { + TString Csv; + TString YsonStr; + + TTestData(TString csv, TString ysonStr) + : Csv(std::move(csv)) + , YsonStr(std::move(ysonStr)) + { + } + }; + + struct TTestDataWithScheme { + TString Scheme; + TVector<TTestData> Data; + + TTestDataWithScheme() = default; + + TTestDataWithScheme(TString&& scheme, TVector<TTestData>&& data) + : Scheme(std::move(scheme)) + , Data(std::move(data)) + { + } + }; + + TTestData GenerateTestData(const TString& keyPrefix, ui32 count) { + TStringBuilder csv; + TStringBuilder yson; + + for (ui32 i = 1; i <= count; ++i) { + // csv + if (keyPrefix) { + csv << "\"" << keyPrefix << i << "\","; + } else { + csv << i << ","; + } + + csv << "\"" << "value" << i << "\"" << Endl; + + // yson + if (i == 1) { + yson << "[[[["; + } else { + yson << ";"; + } + + yson << "[" + << "[\"" << keyPrefix << i << "\"];" + << "[\"" << "value" << i << "\"]" + << "]"; + + if (i == count) { + yson << "];\%false]]]"; + } + } + + return TTestData(std::move(csv), std::move(yson)); + } + + TTestDataWithScheme GenerateTestData(const TString& scheme, const TVector<std::pair<TString, ui64>>& shardsConfig) { + TTestDataWithScheme result; + result.Scheme = scheme; + + for (const auto& [keyPrefix, count] : shardsConfig) { + result.Data.push_back(GenerateTestData(keyPrefix, count)); + } + + return result; + } + + THashMap<TString, TString> ConvertTestData(const THashMap<TString, TTestDataWithScheme>& data) { + THashMap<TString, TString> result; + + for (const auto& [prefix, item] : data) { + result.emplace(prefix + "/scheme.pb", item.Scheme); + for (ui32 i = 0; i < item.Data.size(); ++i) { + result.emplace(Sprintf("%s/data_%02d.csv", prefix.data(), i), item.Data.at(i).Csv); + } + } + + return result; + } + + THashMap<TString, TString> ConvertTestData(const TTestDataWithScheme& data) { + return ConvertTestData({{"", data}}); + } + + NKikimrMiniKQL::TResult ReadTableImpl(TTestActorRuntime& runtime, ui64 tabletId, const TString& query) { + NKikimrMiniKQL::TResult result; + + TString error; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, query, result, error); UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - - return result; - } - - struct TReadKeyDesc { - TString Name; - TString Type; - TString Atom; - }; - - NKikimrMiniKQL::TResult ReadTable(TTestActorRuntime& runtime, ui64 tabletId, - const TString& table = "Table", - const TReadKeyDesc& keyDesc = {"key", "Utf8", "\"\""}, - const TVector<TString>& columns = {"key", "value"}) { - - const auto rangeFmt = Sprintf("'%s (%s '%s)", keyDesc.Name.data(), keyDesc.Type.data(), keyDesc.Atom.data()); - const auto columnsFmt = "'" + JoinSeq(" '", columns); - - return ReadTableImpl(runtime, tabletId, Sprintf(R"( - ( - (let range '( '(%s (Void) ))) - (let columns '(%s) ) - (let result (SelectRange '__user__%s range columns '())) - (return (AsList (SetResult 'Result result) )) - ) - )", rangeFmt.data(), columnsFmt.data(), table.data())); - } - - using TDelayFunc = std::function<bool(TAutoPtr<IEventHandle>&)>; - - auto SetDelayObserver(TTestActorRuntime& runtime, THolder<IEventHandle>& delayed, TDelayFunc delayFunc) { - return runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (delayFunc(ev)) { - delayed.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } - return TTestActorRuntime::EEventAction::PROCESS; - }); - } - - void WaitForDelayed(TTestActorRuntime& runtime, THolder<IEventHandle>& delayed, TTestActorRuntime::TEventObserver prevObserver) { - if (!delayed) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return bool(delayed); - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - } - - -} // anonymous - -Y_UNIT_TEST_SUITE(TRestoreTests) { - void RestoreNoWait(TTestBasicRuntime& runtime, ui64& txId, - ui16 port, THolder<TS3Mock>& s3Mock, TVector<TTestData>&& data, ui32 readBatchSize = 128) { - - const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + + return result; + } + + struct TReadKeyDesc { + TString Name; + TString Type; + TString Atom; + }; + + NKikimrMiniKQL::TResult ReadTable(TTestActorRuntime& runtime, ui64 tabletId, + const TString& table = "Table", + const TReadKeyDesc& keyDesc = {"key", "Utf8", "\"\""}, + const TVector<TString>& columns = {"key", "value"}) { + + const auto rangeFmt = Sprintf("'%s (%s '%s)", keyDesc.Name.data(), keyDesc.Type.data(), keyDesc.Atom.data()); + const auto columnsFmt = "'" + JoinSeq(" '", columns); + + return ReadTableImpl(runtime, tabletId, Sprintf(R"( + ( + (let range '( '(%s (Void) ))) + (let columns '(%s) ) + (let result (SelectRange '__user__%s range columns '())) + (return (AsList (SetResult 'Result result) )) + ) + )", rangeFmt.data(), columnsFmt.data(), table.data())); + } + + using TDelayFunc = std::function<bool(TAutoPtr<IEventHandle>&)>; + + auto SetDelayObserver(TTestActorRuntime& runtime, THolder<IEventHandle>& delayed, TDelayFunc delayFunc) { + return runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (delayFunc(ev)) { + delayed.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + return TTestActorRuntime::EEventAction::PROCESS; + }); + } + + void WaitForDelayed(TTestActorRuntime& runtime, THolder<IEventHandle>& delayed, TTestActorRuntime::TEventObserver prevObserver) { + if (!delayed) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return bool(delayed); + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + } + + +} // anonymous + +Y_UNIT_TEST_SUITE(TRestoreTests) { + void RestoreNoWait(TTestBasicRuntime& runtime, ui64& txId, + ui16 port, THolder<TS3Mock>& s3Mock, TVector<TTestData>&& data, ui32 readBatchSize = 128) { + + const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess); - - s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), std::move(data)}), TS3Mock::TSettings(port))); - UNIT_ASSERT(s3Mock->Start()); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - - TestRestore(runtime, ++txId, "/MyRoot", Sprintf(R"( - TableName: "Table" - TableDescription { - %s - } - S3Settings { - Endpoint: "localhost:%d" - Scheme: HTTP - Limits { - ReadBatchSize: %d - } - } - )", GenerateTableDescription(desc).data(), port, readBatchSize)); - } - - void Restore(TTestBasicRuntime& runtime, TTestEnv& env, const TString& creationScheme, TVector<TTestData>&& data, ui32 readBatchSize = 128) { - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", creationScheme); - env.TestWaitNotification(runtime, txId); - - TPortManager portManager; - THolder<TS3Mock> s3Mock; - - RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, std::move(data), readBatchSize); - env.TestWaitNotification(runtime, txId); - } - - void Restore(TTestBasicRuntime& runtime, const TString& creationScheme, TVector<TTestData>&& data, ui32 readBatchSize = 128) { - TTestEnv env(runtime); - Restore(runtime, env, creationScheme, std::move(data), readBatchSize); - } - - Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData("a", 1); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - + + s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), std::move(data)}), TS3Mock::TSettings(port))); + UNIT_ASSERT(s3Mock->Start()); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + + TestRestore(runtime, ++txId, "/MyRoot", Sprintf(R"( + TableName: "Table" + TableDescription { + %s + } + S3Settings { + Endpoint: "localhost:%d" + Scheme: HTTP + Limits { + ReadBatchSize: %d + } + } + )", GenerateTableDescription(desc).data(), port, readBatchSize)); + } + + void Restore(TTestBasicRuntime& runtime, TTestEnv& env, const TString& creationScheme, TVector<TTestData>&& data, ui32 readBatchSize = 128) { + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", creationScheme); + env.TestWaitNotification(runtime, txId); + + TPortManager portManager; + THolder<TS3Mock> s3Mock; + + RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, std::move(data), readBatchSize); + env.TestWaitNotification(runtime, txId); + } + + void Restore(TTestBasicRuntime& runtime, const TString& creationScheme, TVector<TTestData>&& data, ui32 readBatchSize = 128) { + TTestEnv env(runtime); + Restore(runtime, env, creationScheme, std::move(data), readBatchSize); + } + + Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData("a", 1); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { - TTestBasicRuntime runtime; - - const auto a = GenerateTestData("a", 1); - const auto b = GenerateTestData("b", 1); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - SplitBoundary { - KeyPrefix { - Tuple { Optional { Text: "b" } } - } - } - )", {a, b}); - - { + } + + Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { + TTestBasicRuntime runtime; + + const auto a = GenerateTestData("a", 1); + const auto b = GenerateTestData("b", 1); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { + KeyPrefix { + Tuple { Optional { Text: "b" } } + } + } + )", {a, b}); + + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(a.YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); NKqp::CompareYson(b.YsonStr, content); - } - } - - Y_UNIT_TEST(ShouldSucceedOnLargeData) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData("", 100); - UNIT_ASSERT(data.Csv.size() > 128); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - + } + } + + Y_UNIT_TEST(ShouldSucceedOnLargeData) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData("", 100); + UNIT_ASSERT(data.Csv.size() > 128); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets, "Table", {"key", "Uint32", "0"}); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldSucceedOnSupportedDatatypes) { - TTestBasicRuntime runtime; - - TString csv = TStringBuilder() - << "1," // key - << "-100500," // int32 - << "100500," // uint32 - << "-200500," // int64 - << "200500," // uint64 - << "255," // uint8 - << "1," // bool - << "1.1234," // double - << "-1.123," // float - << "2020-08-12T00:00:00.000000Z," // date - << "2020-08-12T12:34:56.000000Z," // datetime - << "2020-08-12T12:34:56.123456Z," // timestamp - << "-300500," // interval - << "3.321," // decimal - << ".3321e1," // dynumber - << "\"" << CGIEscapeRet("lorem ipsum") << "\"," // string - << "\"" << CGIEscapeRet("lorem ipsum dolor sit amet") << "\"," // utf8 - << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"," // json - << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"" // jsondoc - << Endl; - - TString yson = TStringBuilder() << "[[[[[" - << "[%true];" // bool - << "[\"" << TInstant::ParseIso8601("2020-08-12T00:00:00.000000Z").Days() << "\"];" // date - << "[\"" << TInstant::ParseIso8601("2020-08-12T12:34:56.000000Z").Seconds() << "\"];" // datetime - << "[\"" << "3.321" << "\"];" // decimal - << "[\"" << 1.1234 << "\"];" // double - << "[\"" << ".3321e1" << "\"];" // dynumber - << "[\"" << -1.123f << "\"];" // float - << "[\"" << -100500 << "\"];" // int32 - << "[\"" << -200500 << "\"];" // int64 - << "[\"" << -300500 << "\"];" // interval - << "[\"" << "{\\\"key\\\": \\\"value\\\"}" << "\"];" // json - << "[\"" << "{\\\"key\\\":\\\"value\\\"}" << "\"];" // jsondoc - << "[\"" << 1 << "\"];" // key - << "[\"" << "lorem ipsum" << "\"];" // string - << "[\"" << TInstant::ParseIso8601("2020-08-12T12:34:56.123456Z").MicroSeconds() << "\"];" // timestamp - << "[\"" << 100500 << "\"];" // uint32 - << "[\"" << 200500 << "\"];" // uint64 - << "[\"" << 255 << "\"];" // uint8 - << "[\"" << "lorem ipsum dolor sit amet" << "\"]" // utf8 - << "]];\%false]]]"; - - const auto data = TTestData(std::move(csv), std::move(yson)); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "int32_value" Type: "Int32" } - Columns { Name: "uint32_value" Type: "Uint32" } - Columns { Name: "int64_value" Type: "Int64" } - Columns { Name: "uint64_value" Type: "Uint64" } - Columns { Name: "uint8_value" Type: "Uint8" } - Columns { Name: "bool_value" Type: "Bool" } - Columns { Name: "double_value" Type: "Double" } - Columns { Name: "float_value" Type: "Float" } - Columns { Name: "date_value" Type: "Date" } - Columns { Name: "datetime_value" Type: "Datetime" } - Columns { Name: "timestamp_value" Type: "Timestamp" } - Columns { Name: "interval_value" Type: "Interval" } - Columns { Name: "decimal_value" Type: "Decimal" } - Columns { Name: "dynumber_value" Type: "DyNumber" } - Columns { Name: "string_value" Type: "String" } - Columns { Name: "utf8_value" Type: "Utf8" } - Columns { Name: "json_value" Type: "Json" } - Columns { Name: "jsondoc_value" Type: "JsonDocument" } - KeyColumnNames: ["key"] - )", {data}, data.Csv.size() + 1); - + } + + Y_UNIT_TEST(ShouldSucceedOnSupportedDatatypes) { + TTestBasicRuntime runtime; + + TString csv = TStringBuilder() + << "1," // key + << "-100500," // int32 + << "100500," // uint32 + << "-200500," // int64 + << "200500," // uint64 + << "255," // uint8 + << "1," // bool + << "1.1234," // double + << "-1.123," // float + << "2020-08-12T00:00:00.000000Z," // date + << "2020-08-12T12:34:56.000000Z," // datetime + << "2020-08-12T12:34:56.123456Z," // timestamp + << "-300500," // interval + << "3.321," // decimal + << ".3321e1," // dynumber + << "\"" << CGIEscapeRet("lorem ipsum") << "\"," // string + << "\"" << CGIEscapeRet("lorem ipsum dolor sit amet") << "\"," // utf8 + << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"," // json + << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"" // jsondoc + << Endl; + + TString yson = TStringBuilder() << "[[[[[" + << "[%true];" // bool + << "[\"" << TInstant::ParseIso8601("2020-08-12T00:00:00.000000Z").Days() << "\"];" // date + << "[\"" << TInstant::ParseIso8601("2020-08-12T12:34:56.000000Z").Seconds() << "\"];" // datetime + << "[\"" << "3.321" << "\"];" // decimal + << "[\"" << 1.1234 << "\"];" // double + << "[\"" << ".3321e1" << "\"];" // dynumber + << "[\"" << -1.123f << "\"];" // float + << "[\"" << -100500 << "\"];" // int32 + << "[\"" << -200500 << "\"];" // int64 + << "[\"" << -300500 << "\"];" // interval + << "[\"" << "{\\\"key\\\": \\\"value\\\"}" << "\"];" // json + << "[\"" << "{\\\"key\\\":\\\"value\\\"}" << "\"];" // jsondoc + << "[\"" << 1 << "\"];" // key + << "[\"" << "lorem ipsum" << "\"];" // string + << "[\"" << TInstant::ParseIso8601("2020-08-12T12:34:56.123456Z").MicroSeconds() << "\"];" // timestamp + << "[\"" << 100500 << "\"];" // uint32 + << "[\"" << 200500 << "\"];" // uint64 + << "[\"" << 255 << "\"];" // uint8 + << "[\"" << "lorem ipsum dolor sit amet" << "\"]" // utf8 + << "]];\%false]]]"; + + const auto data = TTestData(std::move(csv), std::move(yson)); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "int32_value" Type: "Int32" } + Columns { Name: "uint32_value" Type: "Uint32" } + Columns { Name: "int64_value" Type: "Int64" } + Columns { Name: "uint64_value" Type: "Uint64" } + Columns { Name: "uint8_value" Type: "Uint8" } + Columns { Name: "bool_value" Type: "Bool" } + Columns { Name: "double_value" Type: "Double" } + Columns { Name: "float_value" Type: "Float" } + Columns { Name: "date_value" Type: "Date" } + Columns { Name: "datetime_value" Type: "Datetime" } + Columns { Name: "timestamp_value" Type: "Timestamp" } + Columns { Name: "interval_value" Type: "Interval" } + Columns { Name: "decimal_value" Type: "Decimal" } + Columns { Name: "dynumber_value" Type: "DyNumber" } + Columns { Name: "string_value" Type: "String" } + Columns { Name: "utf8_value" Type: "Utf8" } + Columns { Name: "json_value" Type: "Json" } + Columns { Name: "jsondoc_value" Type: "JsonDocument" } + KeyColumnNames: ["key"] + )", {data}, data.Csv.size() + 1); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets, "Table", {"key", "Uint64", "0"}, { - "key", - "int32_value", - "uint32_value", - "int64_value", - "uint64_value", - "uint8_value", - "bool_value", - "double_value", - "float_value", - "date_value", - "datetime_value", - "timestamp_value", - "interval_value", - "decimal_value", - "dynumber_value", - "string_value", - "utf8_value", - "json_value", - "jsondoc_value", - }); + "key", + "int32_value", + "uint32_value", + "int64_value", + "uint64_value", + "uint8_value", + "bool_value", + "double_value", + "float_value", + "date_value", + "datetime_value", + "timestamp_value", + "interval_value", + "decimal_value", + "dynumber_value", + "string_value", + "utf8_value", + "json_value", + "jsondoc_value", + }); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldRestoreSpecialFpValues) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock({}, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Original" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "double_value" Type: "Double" } - Columns { Name: "float_value" Type: "Float" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - auto writeRow = [&](ui64 key, double doubleValue, float floatValue) { - NKikimrMiniKQL::TResult result; - TString error; + } + + Y_UNIT_TEST(ShouldRestoreSpecialFpValues) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Original" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "double_value" Type: "Double" } + Columns { Name: "float_value" Type: "Float" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + auto writeRow = [&](ui64 key, double doubleValue, float floatValue) { + NKikimrMiniKQL::TResult result; + TString error; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets, Sprintf(R"( - ( - (let key '( '('key (Uint32 '%lu) ) ) ) - (let row '( '('double_value (Double '%lf ) ) '('float_value (Float '%f) ) ) ) - (return (AsList (UpdateRow '__user__Original key row) )) - ) - )", key, doubleValue, floatValue), result, error); - - UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - }; - - using double_conversion::Double; - - writeRow(1, Double::NaN(), static_cast<float>(Double::NaN())); - writeRow(2, -Double::NaN(), static_cast<float>(-Double::NaN())); - writeRow(3, Double::Infinity(), static_cast<float>(Double::Infinity())); - writeRow(4, -Double::Infinity(), static_cast<float>(-Double::Infinity())); - - TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ExportToS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_path: "/MyRoot/Original" - destination_prefix: "" - } - } - )", port)); - env.TestWaitNotification(runtime, txId); - TestGetExport(runtime, txId, "/MyRoot"); - - TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Restored" - } - } - )", port)); - env.TestWaitNotification(runtime, txId); - TestGetImport(runtime, txId, "/MyRoot"); - } - - Y_UNIT_TEST(ShouldCountWrittenBytesAndRows) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - - const auto data = GenerateTestData("a", 2); - - TMaybe<NKikimrTxDataShard::TShardOpResult> result; - runtime.SetObserverFunc([&result](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() != TEvDataShard::EvSchemaChanged) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - const auto& record = ev->Get<TEvDataShard::TEvSchemaChanged>()->Record; - if (!record.HasOpResult()) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - result = record.GetOpResult(); - return TTestActorRuntime::EEventAction::PROCESS; - }); - - Restore(runtime, env, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - if (!result) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&result](IEventHandle&) -> bool { - return result.Defined(); - }); - runtime.DispatchEvents(opts); - } - - UNIT_ASSERT_VALUES_EQUAL(result->GetBytesProcessed(), 16); - UNIT_ASSERT_VALUES_EQUAL(result->GetRowsProcessed(), 2); - } - - Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { - TTestBasicRuntime runtime; - - const auto data = TTestData("\"a1\",\"value1\"", EmptyYsonStr); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - + ( + (let key '( '('key (Uint32 '%lu) ) ) ) + (let row '( '('double_value (Double '%lf ) ) '('float_value (Float '%f) ) ) ) + (return (AsList (UpdateRow '__user__Original key row) )) + ) + )", key, doubleValue, floatValue), result, error); + + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }; + + using double_conversion::Double; + + writeRow(1, Double::NaN(), static_cast<float>(Double::NaN())); + writeRow(2, -Double::NaN(), static_cast<float>(-Double::NaN())); + writeRow(3, Double::Infinity(), static_cast<float>(Double::Infinity())); + writeRow(4, -Double::Infinity(), static_cast<float>(-Double::Infinity())); + + TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Original" + destination_prefix: "" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + TestGetExport(runtime, txId, "/MyRoot"); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Restored" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + TestGetImport(runtime, txId, "/MyRoot"); + } + + Y_UNIT_TEST(ShouldCountWrittenBytesAndRows) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + + const auto data = GenerateTestData("a", 2); + + TMaybe<NKikimrTxDataShard::TShardOpResult> result; + runtime.SetObserverFunc([&result](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != TEvDataShard::EvSchemaChanged) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + const auto& record = ev->Get<TEvDataShard::TEvSchemaChanged>()->Record; + if (!record.HasOpResult()) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + result = record.GetOpResult(); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Restore(runtime, env, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + if (!result) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&result](IEventHandle&) -> bool { + return result.Defined(); + }); + runtime.DispatchEvents(opts); + } + + UNIT_ASSERT_VALUES_EQUAL(result->GetBytesProcessed(), 16); + UNIT_ASSERT_VALUES_EQUAL(result->GetRowsProcessed(), 2); + } + + Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { + TTestBasicRuntime runtime; + + const auto data = TTestData("\"a1\",\"value1\"", EmptyYsonStr); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldFailOnEmptyToken) { - TTestBasicRuntime runtime; - - const auto data = TTestData("\"a1\",\n", EmptyYsonStr); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - + } + + Y_UNIT_TEST(ShouldFailOnEmptyToken) { + TTestBasicRuntime runtime; + + const auto data = TTestData("\"a1\",\n", EmptyYsonStr); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldFailOnInvalidValue) { - TTestBasicRuntime runtime; - - const auto data = TTestData("\"a1\",\"value1\"\n", EmptyYsonStr); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - + } + + Y_UNIT_TEST(ShouldFailOnInvalidValue) { + TTestBasicRuntime runtime; + + const auto data = TTestData("\"a1\",\"value1\"\n", EmptyYsonStr); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets, "Table", {"key", "Uint64", "0"}); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(ShouldFailOnOutboundKey) { - TTestBasicRuntime runtime; - - const auto a = GenerateTestData("a", 1); - const auto b = TTestData(a.Csv, EmptyYsonStr); - - Restore(runtime, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - SplitBoundary { - KeyPrefix { - Tuple { Optional { Text: "b" } } - } - } - )", {a, b}); - - { + } + + Y_UNIT_TEST(ShouldFailOnOutboundKey) { + TTestBasicRuntime runtime; + + const auto a = GenerateTestData("a", 1); + const auto b = TTestData(a.Csv, EmptyYsonStr); + + Restore(runtime, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { + KeyPrefix { + Tuple { Optional { Text: "b" } } + } + } + )", {a, b}); + + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(a.YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); NKqp::CompareYson(b.YsonStr, content); - } - } - + } + } + void TestRestoreNegative(TTestActorRuntime& runtime, ui64 txId, const TString& parentPath, const TString& name, const TVector<TEvSchemeShard::EStatus>& expectedResults) { - - TestRestore(runtime, ++txId, parentPath, Sprintf(R"( - TableName: "%s" - S3Settings { - Endpoint: "localhost" - Scheme: HTTP - } - )", name.data()), expectedResults); - } - - Y_UNIT_TEST(ShouldFailOnVariousErrors) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); + + TestRestore(runtime, ++txId, parentPath, Sprintf(R"( + TableName: "%s" + S3Settings { + Endpoint: "localhost" + Scheme: HTTP + } + )", name.data()), expectedResults); + } + + Y_UNIT_TEST(ShouldFailOnVariousErrors) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); TestRestoreNegative(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusMultipleModifications}); - env.TestWaitNotification(runtime, {txId - 1, txId}); - - TestMkDir(runtime, ++txId, "/MyRoot", "Dir"); + env.TestWaitNotification(runtime, {txId - 1, txId}); + + TestMkDir(runtime, ++txId, "/MyRoot", "Dir"); TestRestoreNegative(runtime, ++txId, "/MyRoot", "Dir", {NKikimrScheme::StatusNameConflict}); - env.TestWaitNotification(runtime, {txId - 1, txId}); - + env.TestWaitNotification(runtime, {txId - 1, txId}); + TestRestoreNegative(runtime, ++txId, "/MyRoot", "Dir", {NKikimrScheme::StatusNameConflict}); TestRestoreNegative(runtime, ++txId, "/MyRoot", "NotExist", {NKikimrScheme::StatusPathDoesNotExist}); - - TestAlterTable(runtime, ++txId, "/MyRoot",R"( - Name: "Table" - Columns { Name: "extra" Type: "Utf8"} - )"); + + TestAlterTable(runtime, ++txId, "/MyRoot",R"( + Name: "Table" + Columns { Name: "extra" Type: "Utf8"} + )"); TestRestoreNegative(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusMultipleModifications}); - env.TestWaitNotification(runtime, {txId - 1, txId}); - - TestDropTable(runtime, ++txId, "/MyRoot", "Table"); + env.TestWaitNotification(runtime, {txId - 1, txId}); + + TestDropTable(runtime, ++txId, "/MyRoot", "Table"); TestRestoreNegative(runtime, ++txId, "/MyRoot", "Table", {NKikimrScheme::StatusMultipleModifications}); - env.TestWaitNotification(runtime, {txId - 1, txId}); - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "IndexedTable" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "ByValue" - KeyColumnNames: ["value"] - } - )"); + env.TestWaitNotification(runtime, {txId - 1, txId}); + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "IndexedTable" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ByValue" + KeyColumnNames: ["value"] + } + )"); TestRestoreNegative(runtime, ++txId, "/MyRoot", "IndexedTable", {NKikimrScheme::StatusMultipleModifications}); - env.TestWaitNotification(runtime, {txId - 1, txId}); - + env.TestWaitNotification(runtime, {txId - 1, txId}); + TestRestoreNegative(runtime, ++txId, "/MyRoot", "IndexedTable", {NKikimrScheme::StatusInvalidParameter}); - } - - template <typename TEvToDelay> - void CancelShouldSucceed(const TTestData& data, bool kill = false) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - THolder<IEventHandle> delayed; - auto prevObserver = SetDelayObserver(runtime, delayed, [](TAutoPtr<IEventHandle>& ev) { - return ev->GetTypeRewrite() == TEvToDelay::EventType; - }); - - TPortManager portManager; - THolder<TS3Mock> s3Mock; - - runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); - RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, {data}); - const ui64 restoreTxId = txId; - - if (kill) { - s3Mock.Destroy(); - } - - WaitForDelayed(runtime, delayed, prevObserver); - - runtime.Send(delayed.Release(), 0, true); - TestCancelTxTable(runtime, ++txId, restoreTxId); - env.TestWaitNotification(runtime, {restoreTxId, txId}); - + } + + template <typename TEvToDelay> + void CancelShouldSucceed(const TTestData& data, bool kill = false) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + THolder<IEventHandle> delayed; + auto prevObserver = SetDelayObserver(runtime, delayed, [](TAutoPtr<IEventHandle>& ev) { + return ev->GetTypeRewrite() == TEvToDelay::EventType; + }); + + TPortManager portManager; + THolder<TS3Mock> s3Mock; + + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, {data}); + const ui64 restoreTxId = txId; + + if (kill) { + s3Mock.Destroy(); + } + + WaitForDelayed(runtime, delayed, prevObserver); + + runtime.Send(delayed.Release(), 0, true); + TestCancelTxTable(runtime, ++txId, restoreTxId); + env.TestWaitNotification(runtime, {restoreTxId, txId}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - - Y_UNIT_TEST(CancelUponProposeShouldSucceed) { - auto data = GenerateTestData("a", 1); - data.YsonStr = EmptyYsonStr; - CancelShouldSucceed<TEvDataShard::TEvProposeTransaction>(data); - } - - Y_UNIT_TEST(CancelUponProposeResultShouldSucceed) { - auto data = GenerateTestData("a", 1); - data.YsonStr = EmptyYsonStr; - CancelShouldSucceed<TEvDataShard::TEvProposeTransactionResult>(data); - } - - Y_UNIT_TEST(CancelUponUploadResponseShouldSucceed) { - const auto data = GenerateTestData("a", 1); - CancelShouldSucceed<TEvDataShard::TEvUnsafeUploadRowsResponse>(data); - } - - Y_UNIT_TEST(CancelHungOperationShouldSucceed) { - auto data = GenerateTestData("a", 1); - data.YsonStr = EmptyYsonStr; - CancelShouldSucceed<TEvDataShard::TEvProposeTransactionResult>(data, true); - } - - Y_UNIT_TEST(CancelAlmostCompleteOperationShouldNotHaveEffect) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - THolder<IEventHandle> schemaChanged; - auto prevObserver = SetDelayObserver(runtime, schemaChanged, [](TAutoPtr<IEventHandle>& ev) { - return ev->GetTypeRewrite() == TEvDataShard::TEvSchemaChanged::EventType; - }); - - TPortManager portManager; - THolder<TS3Mock> s3Mock; - const auto data = GenerateTestData("a", 1); - - RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, {data}); - const ui64 restoreTxId = txId; - - WaitForDelayed(runtime, schemaChanged, prevObserver); - - THolder<IEventHandle> progress; - prevObserver = SetDelayObserver(runtime, progress, [](TAutoPtr<IEventHandle>& ev) { - return ev->GetTypeRewrite() == TEvPrivate::TEvProgressOperation::EventType; - }); - - TestCancelTxTable(runtime, ++txId, restoreTxId); - WaitForDelayed(runtime, progress, prevObserver); - - runtime.Send(schemaChanged.Release(), 0, true); - runtime.Send(progress.Release(), 0, true); - env.TestWaitNotification(runtime, {restoreTxId, txId}); - + } + + Y_UNIT_TEST(CancelUponProposeShouldSucceed) { + auto data = GenerateTestData("a", 1); + data.YsonStr = EmptyYsonStr; + CancelShouldSucceed<TEvDataShard::TEvProposeTransaction>(data); + } + + Y_UNIT_TEST(CancelUponProposeResultShouldSucceed) { + auto data = GenerateTestData("a", 1); + data.YsonStr = EmptyYsonStr; + CancelShouldSucceed<TEvDataShard::TEvProposeTransactionResult>(data); + } + + Y_UNIT_TEST(CancelUponUploadResponseShouldSucceed) { + const auto data = GenerateTestData("a", 1); + CancelShouldSucceed<TEvDataShard::TEvUnsafeUploadRowsResponse>(data); + } + + Y_UNIT_TEST(CancelHungOperationShouldSucceed) { + auto data = GenerateTestData("a", 1); + data.YsonStr = EmptyYsonStr; + CancelShouldSucceed<TEvDataShard::TEvProposeTransactionResult>(data, true); + } + + Y_UNIT_TEST(CancelAlmostCompleteOperationShouldNotHaveEffect) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + THolder<IEventHandle> schemaChanged; + auto prevObserver = SetDelayObserver(runtime, schemaChanged, [](TAutoPtr<IEventHandle>& ev) { + return ev->GetTypeRewrite() == TEvDataShard::TEvSchemaChanged::EventType; + }); + + TPortManager portManager; + THolder<TS3Mock> s3Mock; + const auto data = GenerateTestData("a", 1); + + RestoreNoWait(runtime, txId, portManager.GetPort(), s3Mock, {data}); + const ui64 restoreTxId = txId; + + WaitForDelayed(runtime, schemaChanged, prevObserver); + + THolder<IEventHandle> progress; + prevObserver = SetDelayObserver(runtime, progress, [](TAutoPtr<IEventHandle>& ev) { + return ev->GetTypeRewrite() == TEvPrivate::TEvProgressOperation::EventType; + }); + + TestCancelTxTable(runtime, ++txId, restoreTxId); + WaitForDelayed(runtime, progress, prevObserver); + + runtime.Send(schemaChanged.Release(), 0, true); + runtime.Send(progress.Release(), 0, true); + env.TestWaitNotification(runtime, {restoreTxId, txId}); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } -} - -Y_UNIT_TEST_SUITE(TRestoreWithRebootsTests) { - void Restore(TTestWithReboots& t, TTestActorRuntime& runtime, bool& activeZone, - ui16 port, const TString& creationScheme, TVector<TTestData>&& data) { - - THolder<TS3Mock> s3Mock; - TString schemeStr; - - { - TInactiveZone inactive(activeZone); - - TestCreateTable(runtime, ++t.TxId, "/MyRoot", creationScheme); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); + } +} + +Y_UNIT_TEST_SUITE(TRestoreWithRebootsTests) { + void Restore(TTestWithReboots& t, TTestActorRuntime& runtime, bool& activeZone, + ui16 port, const TString& creationScheme, TVector<TTestData>&& data) { + + THolder<TS3Mock> s3Mock; + TString schemeStr; + + { + TInactiveZone inactive(activeZone); + + TestCreateTable(runtime, ++t.TxId, "/MyRoot", creationScheme); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess); - - s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), std::move(data)}), TS3Mock::TSettings(port))); - UNIT_ASSERT(s3Mock->Start()); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - schemeStr = GenerateTableDescription(desc); - } - - TestRestore(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( - TableName: "Table" - TableDescription { - %s - } - S3Settings { - Endpoint: "localhost:%d" - Scheme: HTTP - Limits { - ReadBatchSize: 128 - } - } - )", schemeStr.data(), port)); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto data = GenerateTestData("a", 1); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - { - TInactiveZone inactive(activeZone); - + + s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), std::move(data)}), TS3Mock::TSettings(port))); + UNIT_ASSERT(s3Mock->Start()); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + schemeStr = GenerateTableDescription(desc); + } + + TestRestore(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( + TableName: "Table" + TableDescription { + %s + } + S3Settings { + Endpoint: "localhost:%d" + Scheme: HTTP + Limits { + ReadBatchSize: 128 + } + } + )", schemeStr.data(), port)); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto data = GenerateTestData("a", 1); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + { + TInactiveZone inactive(activeZone); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - }); - } - - Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto a = GenerateTestData("a", 1); - const auto b = GenerateTestData("b", 1); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - SplitBoundary { - KeyPrefix { - Tuple { Optional { Text: "b" } } - } - } - )", {a, b}); - - { - TInactiveZone inactive(activeZone); - { + } + }); + } + + Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto a = GenerateTestData("a", 1); + const auto b = GenerateTestData("b", 1); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { + KeyPrefix { + Tuple { Optional { Text: "b" } } + } + } + )", {a, b}); + + { + TInactiveZone inactive(activeZone); + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(a.YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); NKqp::CompareYson(b.YsonStr, content); - } - } - }); - } - - Y_UNIT_TEST(ShouldSucceedOnMultiShardTableAndLimitedResources) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - using namespace NResourceBroker; - - auto config = MakeDefaultConfig(); - for (auto& queue : *config.MutableQueues()) { - if (queue.GetName() == "queue_restore") { - queue.MutableLimit()->SetCpu(1); - break; - } - } - - runtime.RegisterService(MakeResourceBrokerID(), - runtime.Register(CreateResourceBrokerActor(config, runtime.GetDynamicCounters(0)))); - } - - const auto a = GenerateTestData("a", 1); - const auto b = GenerateTestData("b", 1); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - SplitBoundary { - KeyPrefix { - Tuple { Optional { Text: "b" } } - } - } - )", {a, b}); - - { - TInactiveZone inactive(activeZone); - { + } + } + }); + } + + Y_UNIT_TEST(ShouldSucceedOnMultiShardTableAndLimitedResources) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + using namespace NResourceBroker; + + auto config = MakeDefaultConfig(); + for (auto& queue : *config.MutableQueues()) { + if (queue.GetName() == "queue_restore") { + queue.MutableLimit()->SetCpu(1); + break; + } + } + + runtime.RegisterService(MakeResourceBrokerID(), + runtime.Register(CreateResourceBrokerActor(config, runtime.GetDynamicCounters(0)))); + } + + const auto a = GenerateTestData("a", 1); + const auto b = GenerateTestData("b", 1); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { + KeyPrefix { + Tuple { Optional { Text: "b" } } + } + } + )", {a, b}); + + { + TInactiveZone inactive(activeZone); + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); - NKqp::CompareYson(a.YsonStr, content); - } - { + NKqp::CompareYson(a.YsonStr, content); + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); - NKqp::CompareYson(b.YsonStr, content); - } - } - }); - } - - Y_UNIT_TEST(ShouldSucceedOnLargeData) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto data = GenerateTestData("", 100); - UNIT_ASSERT(data.Csv.size() > 128); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Uint32" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - { - TInactiveZone inactive(activeZone); - + NKqp::CompareYson(b.YsonStr, content); + } + } + }); + } + + Y_UNIT_TEST(ShouldSucceedOnLargeData) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto data = GenerateTestData("", 100); + UNIT_ASSERT(data.Csv.size() > 128); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + { + TInactiveZone inactive(activeZone); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets, "Table", {"key", "Uint32", "0"}); NKqp::CompareYson(data.YsonStr, content); - } - }); - } - - Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto data = TTestData("\"a1\",\"value1\"", EmptyYsonStr); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - { - TInactiveZone inactive(activeZone); - + } + }); + } + + Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto data = TTestData("\"a1\",\"value1\"", EmptyYsonStr); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + { + TInactiveZone inactive(activeZone); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - }); - } - - Y_UNIT_TEST(ShouldFailOnEmptyToken) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto data = TTestData("\"a1\",\n", EmptyYsonStr); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - { - TInactiveZone inactive(activeZone); - + } + }); + } + + Y_UNIT_TEST(ShouldFailOnEmptyToken) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto data = TTestData("\"a1\",\n", EmptyYsonStr); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + { + TInactiveZone inactive(activeZone); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.YsonStr, content); - } - }); - } - - Y_UNIT_TEST(ShouldFailOnInvalidValue) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto data = TTestData("\"a1\",\"value1\"\n", EmptyYsonStr); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )", {data}); - - { - TInactiveZone inactive(activeZone); - + } + }); + } + + Y_UNIT_TEST(ShouldFailOnInvalidValue) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto data = TTestData("\"a1\",\"value1\"\n", EmptyYsonStr); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", {data}); + + { + TInactiveZone inactive(activeZone); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets, "Table", {"key", "Uint64", "0"}); NKqp::CompareYson(data.YsonStr, content); - } - }); - } - - Y_UNIT_TEST(ShouldFailOnOutboundKey) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - const auto a = GenerateTestData("a", 1); - const auto b = TTestData(a.Csv, EmptyYsonStr); - - Restore(t, runtime, activeZone, port, R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - SplitBoundary { - KeyPrefix { - Tuple { Optional { Text: "b" } } - } - } - )", {a, b}); - - { - TInactiveZone inactive(activeZone); - { + } + }); + } + + Y_UNIT_TEST(ShouldFailOnOutboundKey) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + const auto a = GenerateTestData("a", 1); + const auto b = TTestData(a.Csv, EmptyYsonStr); + + Restore(t, runtime, activeZone, port, R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { + KeyPrefix { + Tuple { Optional { Text: "b" } } + } + } + )", {a, b}); + + { + TInactiveZone inactive(activeZone); + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(a.YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); NKqp::CompareYson(b.YsonStr, content); - } - } - }); - } - - Y_UNIT_TEST(CancelShouldSucceed) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - const auto data = GenerateTestData("a", 1); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - THolder<TS3Mock> s3Mock; - TString schemeStr; - - { - TInactiveZone inactive(activeZone); - - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "Table" - Columns { Name: "key" Type: "Utf8" } - Columns { Name: "value" Type: "Utf8" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); + } + } + }); + } + + Y_UNIT_TEST(CancelShouldSucceed) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + const auto data = GenerateTestData("a", 1); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + THolder<TS3Mock> s3Mock; + TString schemeStr; + + { + TInactiveZone inactive(activeZone); + + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true); UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess); - - s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), {data}}), TS3Mock::TSettings(port))); - UNIT_ASSERT(s3Mock->Start()); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - schemeStr = GenerateTableDescription(desc); - } - - AsyncRestore(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( - TableName: "Table" - TableDescription { - %s - } - S3Settings { - Endpoint: "localhost:%d" - Scheme: HTTP - Limits { - ReadBatchSize: 128 - } - } - )", schemeStr.data(), port)); - const ui64 restoreTxId = t.TxId; - - t.TestEnv->ReliablePropose(runtime, CancelTxRequest(++t.TxId, restoreTxId), { + + s3Mock.Reset(new TS3Mock(ConvertTestData({GenerateScheme(desc), {data}}), TS3Mock::TSettings(port))); + UNIT_ASSERT(s3Mock->Start()); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + schemeStr = GenerateTableDescription(desc); + } + + AsyncRestore(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( + TableName: "Table" + TableDescription { + %s + } + S3Settings { + Endpoint: "localhost:%d" + Scheme: HTTP + Limits { + ReadBatchSize: 128 + } + } + )", schemeStr.data(), port)); + const ui64 restoreTxId = t.TxId; + + t.TestEnv->ReliablePropose(runtime, CancelTxRequest(++t.TxId, restoreTxId), { NKikimrScheme::StatusAccepted, NKikimrScheme::StatusTxIdNotExists - }); - t.TestEnv->TestWaitNotification(runtime, {restoreTxId, t.TxId}); - }); - } -} - -Y_UNIT_TEST_SUITE(TImportTests) { - void Run(TTestBasicRuntime& runtime, TTestEnv& env, - THashMap<TString, TString>&& data, const TString& request, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { - - ui64 id = 100; - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock(data, TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - + }); + t.TestEnv->TestWaitNotification(runtime, {restoreTxId, t.TxId}); + }); + } +} + +Y_UNIT_TEST_SUITE(TImportTests) { + void Run(TTestBasicRuntime& runtime, TTestEnv& env, + THashMap<TString, TString>&& data, const TString& request, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, + const TString& dbName = "/MyRoot", bool serverless = false) { + + ui64 id = 100; + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(data, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + ui64 schemeshardId = TTestTxConfig::SchemeShard; - if (dbName != "/MyRoot") { - TestCreateExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( - Name: "%s" - )", TStringBuf(serverless ? "/MyRoot/Shared" : dbName).RNextTok('/').data())); - env.TestWaitNotification(runtime, id); - - TestAlterExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( - PlanResolution: 50 - Coordinators: 1 - Mediators: 1 - TimeCastBucketsPerMediator: 2 - ExternalSchemeShard: true - Name: "%s" - StoragePools { - Name: "name_User_kind_hdd-1" - Kind: "common" - } - StoragePools { - Name: "name_User_kind_hdd-2" - Kind: "external" - } - )", TStringBuf(serverless ? "/MyRoot/Shared" : dbName).RNextTok('/').data())); - env.TestWaitNotification(runtime, id); - - if (serverless) { - const auto attrs = AlterUserAttrs({ - {"cloud_id", "CLOUD_ID_VAL"}, - {"folder_id", "FOLDER_ID_VAL"}, - {"database_id", "DATABASE_ID_VAL"} - }); - - TestCreateExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( - Name: "%s" - ResourcesDomainKey { - SchemeShard: %lu - PathId: 2 - } + if (dbName != "/MyRoot") { + TestCreateExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( + Name: "%s" + )", TStringBuf(serverless ? "/MyRoot/Shared" : dbName).RNextTok('/').data())); + env.TestWaitNotification(runtime, id); + + TestAlterExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( + PlanResolution: 50 + Coordinators: 1 + Mediators: 1 + TimeCastBucketsPerMediator: 2 + ExternalSchemeShard: true + Name: "%s" + StoragePools { + Name: "name_User_kind_hdd-1" + Kind: "common" + } + StoragePools { + Name: "name_User_kind_hdd-2" + Kind: "external" + } + )", TStringBuf(serverless ? "/MyRoot/Shared" : dbName).RNextTok('/').data())); + env.TestWaitNotification(runtime, id); + + if (serverless) { + const auto attrs = AlterUserAttrs({ + {"cloud_id", "CLOUD_ID_VAL"}, + {"folder_id", "FOLDER_ID_VAL"}, + {"database_id", "DATABASE_ID_VAL"} + }); + + TestCreateExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( + Name: "%s" + ResourcesDomainKey { + SchemeShard: %lu + PathId: 2 + } )", TStringBuf(dbName).RNextTok('/').data(), TTestTxConfig::SchemeShard), attrs); - env.TestWaitNotification(runtime, id); - - TestAlterExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( - PlanResolution: 50 - Coordinators: 1 - Mediators: 1 - TimeCastBucketsPerMediator: 2 - ExternalSchemeShard: true - ExternalHive: false - Name: "%s" + env.TestWaitNotification(runtime, id); + + TestAlterExtSubDomain(runtime, ++id, "/MyRoot", Sprintf(R"( + PlanResolution: 50 + Coordinators: 1 + Mediators: 1 + TimeCastBucketsPerMediator: 2 + ExternalSchemeShard: true + ExternalHive: false + Name: "%s" StoragePools { Name: "name_User_kind_hdd-1" Kind: "common" @@ -1189,906 +1189,906 @@ Y_UNIT_TEST_SUITE(TImportTests) { Name: "name_User_kind_hdd-2" Kind: "external" } - )", TStringBuf(dbName).RNextTok('/').data())); - env.TestWaitNotification(runtime, id); - } - - TestDescribeResult(DescribePath(runtime, dbName), { - NLs::PathExist, - NLs::ExtractTenantSchemeshard(&schemeshardId) - }); - } - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - - TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port)); - env.TestWaitNotification(runtime, id, schemeshardId); - - TestGetImport(runtime, schemeshardId, id, dbName, expectedStatus); - } - - void Run(TTestBasicRuntime& runtime, THashMap<TString, TString>&& data, const TString& request, - Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { - + )", TStringBuf(dbName).RNextTok('/').data())); + env.TestWaitNotification(runtime, id); + } + + TestDescribeResult(DescribePath(runtime, dbName), { + NLs::PathExist, + NLs::ExtractTenantSchemeshard(&schemeshardId) + }); + } + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + + TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port)); + env.TestWaitNotification(runtime, id, schemeshardId); + + TestGetImport(runtime, schemeshardId, id, dbName, expectedStatus); + } + + void Run(TTestBasicRuntime& runtime, THashMap<TString, TString>&& data, const TString& request, + Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, + const TString& dbName = "/MyRoot", bool serverless = false) { + TTestEnv env(runtime, TTestEnvOptions()); - Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless); - } - - Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )"); - + Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless); + } + + Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"); + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); NKqp::CompareYson(data.Data[0].YsonStr, content); - } - - Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - partition_at_keys { - split_points { - type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } - value { items { text_value: "b" } } - } - } - )", {{"a", 1}, {"b", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )"); - - { + } + + Y_UNIT_TEST(ShouldSucceedOnMultiShardTable) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "b" } } + } + } + )", {{"a", 1}, {"b", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"); + + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(data.Data[0].YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1); NKqp::CompareYson(data.Data[1].YsonStr, content); - } - } - - void ShouldSucceedOnIndexedTable(ui32 indexes, const TString& indexType = "global_index {}") { - TTestBasicRuntime runtime; - - auto scheme = TStringBuilder() << R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )"; - - for (ui32 i = 0; i < indexes; ++i) { - scheme << Sprintf(R"( - indexes { - name: "by_value_%i" - index_columns: "value" - %s - } - )", i + 1, indexType.data()); - } - - const auto data = GenerateTestData(scheme, {{"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )"); - - { + } + } + + void ShouldSucceedOnIndexedTable(ui32 indexes, const TString& indexType = "global_index {}") { + TTestBasicRuntime runtime; + + auto scheme = TStringBuilder() << R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )"; + + for (ui32 i = 0; i < indexes; ++i) { + scheme << Sprintf(R"( + indexes { + name: "by_value_%i" + index_columns: "value" + %s + } + )", i + 1, indexType.data()); + } + + const auto data = GenerateTestData(scheme, {{"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"); + + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0); NKqp::CompareYson(data.Data[0].YsonStr, content); - } - - for (ui32 i = 0; i < indexes; ++i) { + } + + for (ui32 i = 0; i < indexes; ++i) { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1 + i, - "indexImplTable", {"value", "Utf8", "\"\""}, {"value", "key"}); + "indexImplTable", {"value", "Utf8", "\"\""}, {"value", "key"}); NKqp::CompareYson(data.Data[0].YsonStr, content); - } - } - - Y_UNIT_TEST(ShouldSucceedOnIndexedTable1) { - ShouldSucceedOnIndexedTable(1); - } - - Y_UNIT_TEST(ShouldSucceedOnIndexedTable2) { - ShouldSucceedOnIndexedTable(2); - } - - Y_UNIT_TEST(ShouldSucceedOnIndexedTable3) { - ShouldSucceedOnIndexedTable(1, ""); - } - - Y_UNIT_TEST(ShouldSucceedOnManyTables) { - TTestBasicRuntime runtime; - - const auto a = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}); - - const auto b = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"b", 1}}); - - Run(runtime, ConvertTestData({{"/a", a}, {"/b", b}}), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "a" - destination_path: "/MyRoot/TableA" - } - items { - source_prefix: "b" - destination_path: "/MyRoot/TableB" - } - } - )"); - - { + } + } + + Y_UNIT_TEST(ShouldSucceedOnIndexedTable1) { + ShouldSucceedOnIndexedTable(1); + } + + Y_UNIT_TEST(ShouldSucceedOnIndexedTable2) { + ShouldSucceedOnIndexedTable(2); + } + + Y_UNIT_TEST(ShouldSucceedOnIndexedTable3) { + ShouldSucceedOnIndexedTable(1, ""); + } + + Y_UNIT_TEST(ShouldSucceedOnManyTables) { + TTestBasicRuntime runtime; + + const auto a = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + const auto b = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"b", 1}}); + + Run(runtime, ConvertTestData({{"/a", a}, {"/b", b}}), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "a" + destination_path: "/MyRoot/TableA" + } + items { + source_prefix: "b" + destination_path: "/MyRoot/TableB" + } + } + )"); + + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 0, "TableA"); NKqp::CompareYson(a.Data[0].YsonStr, content); - } - { + } + { auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets + 1, "TableB"); NKqp::CompareYson(b.Data[0].YsonStr, content); - } - } - - Y_UNIT_TEST(ShouldWriteBillRecordOnServerlessDb) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 100}}); - - TVector<TString> billRecords; - runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); - return TTestActorRuntime::EEventAction::PROCESS; - }); - - Run(runtime, env, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/User/Table" - } - } - )", Ydb::StatusIds::SUCCESS, "/MyRoot/User", true); - - if (billRecords.empty()) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&billRecords](IEventHandle&) -> bool { - return !billRecords.empty(); - }); - runtime.DispatchEvents(opts); - } - + } + } + + Y_UNIT_TEST(ShouldWriteBillRecordOnServerlessDb) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 100}}); + + TVector<TString> billRecords; + runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(runtime, env, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/User/Table" + } + } + )", Ydb::StatusIds::SUCCESS, "/MyRoot/User", true); + + if (billRecords.empty()) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&billRecords](IEventHandle&) -> bool { + return !billRecords.empty(); + }); + runtime.DispatchEvents(opts); + } + const TString expectedBillRecord = R"({"usage":{"start":0,"quantity":50,"finish":0,"unit":"request_unit","type":"delta"},"tags":{},"id":"281474976725758-9437199-2-9437199-4","cloud_id":"CLOUD_ID_VAL","source_wt":0,"source_id":"sless-docapi-ydb-ss","resource_id":"DATABASE_ID_VAL","schema":"ydb.serverless.requests.v1","folder_id":"FOLDER_ID_VAL","version":"1.0.0"})"; - - UNIT_ASSERT_VALUES_EQUAL(billRecords.size(), 1); - UNIT_ASSERT_NO_DIFF(billRecords[0], expectedBillRecord + "\n"); - } - - Y_UNIT_TEST(ShouldNotWriteBillRecordOnCommonDb) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 100}}); - - TVector<TString> billRecords; - runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { - return TTestActorRuntime::EEventAction::PROCESS; - } - - billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); - return TTestActorRuntime::EEventAction::PROCESS; - }); - - Run(runtime, env, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/User/Table" - } - } - )", Ydb::StatusIds::SUCCESS, "/MyRoot/User"); - - UNIT_ASSERT(billRecords.empty()); - } - - void ShouldRestoreSettings(const TString& settings, const TVector<NLs::TCheckFunc>& checks) { - TTestBasicRuntime runtime; - - const auto empty = TTestData("", EmptyYsonStr); - const auto data = TTestDataWithScheme(TStringBuilder() << R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "created_at" - type { optional_type { item { type_id: TIMESTAMP } } } - } - columns { - name: "modified_at" - type { optional_type { item { type_id: UINT32 } } } - } - primary_key: "key" - )" << settings, {empty, empty}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/User/Table" - } - } - )", Ydb::StatusIds::SUCCESS, "/MyRoot/User"); - - ui64 schemeshardId = 0; - TestDescribeResult(DescribePath(runtime, "/MyRoot/User"), { - NLs::PathExist, - NLs::ExtractTenantSchemeshard(&schemeshardId) - }); - - TestDescribeResult(DescribePath(runtime, schemeshardId, "/MyRoot/User/Table", true, true), checks); - } - - Y_UNIT_TEST(ShouldRestoreTtlSettingsInDateTypeColumnMode) { - ShouldRestoreSettings(R"( - ttl_settings { - date_type_column { - column_name: "created_at" - expire_after_seconds: 3600 - } - } - )", { - NLs::HasTtlEnabled("created_at", TDuration::Hours(1)), - }); - } - - Y_UNIT_TEST(ShouldRestoreTtlSettingsInValueSinceUnixEpochMode) { - ShouldRestoreSettings(R"( - ttl_settings { - value_since_unix_epoch { - column_name: "modified_at" - column_unit: UNIT_SECONDS - expire_after_seconds: 7200 - } - } - )", { + + UNIT_ASSERT_VALUES_EQUAL(billRecords.size(), 1); + UNIT_ASSERT_NO_DIFF(billRecords[0], expectedBillRecord + "\n"); + } + + Y_UNIT_TEST(ShouldNotWriteBillRecordOnCommonDb) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 100}}); + + TVector<TString> billRecords; + runtime.SetObserverFunc([&billRecords](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(runtime, env, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/User/Table" + } + } + )", Ydb::StatusIds::SUCCESS, "/MyRoot/User"); + + UNIT_ASSERT(billRecords.empty()); + } + + void ShouldRestoreSettings(const TString& settings, const TVector<NLs::TCheckFunc>& checks) { + TTestBasicRuntime runtime; + + const auto empty = TTestData("", EmptyYsonStr); + const auto data = TTestDataWithScheme(TStringBuilder() << R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "created_at" + type { optional_type { item { type_id: TIMESTAMP } } } + } + columns { + name: "modified_at" + type { optional_type { item { type_id: UINT32 } } } + } + primary_key: "key" + )" << settings, {empty, empty}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/User/Table" + } + } + )", Ydb::StatusIds::SUCCESS, "/MyRoot/User"); + + ui64 schemeshardId = 0; + TestDescribeResult(DescribePath(runtime, "/MyRoot/User"), { + NLs::PathExist, + NLs::ExtractTenantSchemeshard(&schemeshardId) + }); + + TestDescribeResult(DescribePath(runtime, schemeshardId, "/MyRoot/User/Table", true, true), checks); + } + + Y_UNIT_TEST(ShouldRestoreTtlSettingsInDateTypeColumnMode) { + ShouldRestoreSettings(R"( + ttl_settings { + date_type_column { + column_name: "created_at" + expire_after_seconds: 3600 + } + } + )", { + NLs::HasTtlEnabled("created_at", TDuration::Hours(1)), + }); + } + + Y_UNIT_TEST(ShouldRestoreTtlSettingsInValueSinceUnixEpochMode) { + ShouldRestoreSettings(R"( + ttl_settings { + value_since_unix_epoch { + column_name: "modified_at" + column_unit: UNIT_SECONDS + expire_after_seconds: 7200 + } + } + )", { NLs::HasTtlEnabled("modified_at", TDuration::Hours(2), NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS), - }); - } - - Y_UNIT_TEST(ShouldRestoreStorageSettings) { + }); + } + + Y_UNIT_TEST(ShouldRestoreStorageSettings) { auto check = [](const NKikimrScheme::TEvDescribeSchemeResult& desc) { - const auto& config = desc.GetPathDescription().GetTable().GetPartitionConfig().GetColumnFamilies(0).GetStorageConfig(); - - UNIT_ASSERT_VALUES_EQUAL(config.GetSysLog().GetPreferredPoolKind(), "common"); - UNIT_ASSERT_VALUES_EQUAL(config.GetLog().GetPreferredPoolKind(), "common"); - UNIT_ASSERT_VALUES_EQUAL(config.GetExternal().GetPreferredPoolKind(), "external"); - UNIT_ASSERT(config.HasExternalThreshold()); - }; - - ShouldRestoreSettings(R"( - storage_settings { - tablet_commit_log0 { media: "common" } - tablet_commit_log1 { media: "common" } - external { media: "external" } - store_external_blobs: ENABLED - } - )", { - check, - }); - } - - Y_UNIT_TEST(ShouldRestoreColumnFamilies) { - ShouldRestoreSettings(R"( - storage_settings { - tablet_commit_log0 { media: "common" } - tablet_commit_log1 { media: "common" } - } - column_families { - name: "compressed" - data { media: "common" } - compression: COMPRESSION_LZ4 - } - )", { - NLs::ColumnFamiliesHas(1, "compressed"), - }); - } - - Y_UNIT_TEST(ShouldRestoreAttributes) { - ShouldRestoreSettings(R"( - attributes { - key: "key" - value: "value" - } - )", { - NLs::UserAttrsEqual({{"key", "value"}}), - }); - } - - // Skip compaction_policy (not supported) - // Skip uniform_partitions (has no effect) - - Y_UNIT_TEST(ShouldRestoreSplitPoints) { - ShouldRestoreSettings(R"( - partition_at_keys { - split_points { - type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } - value { items { text_value: "b" } } - } - } - )", { - NLs::CheckBoundaries, - }); - } - - Y_UNIT_TEST(ShouldRestorePartitioningBySize) { - ShouldRestoreSettings(R"( - partitioning_settings { - partitioning_by_size: ENABLED - partition_size_mb: 1024 - } - )", { - NLs::SizeToSplitEqual(1 << 30), - }); - } - - Y_UNIT_TEST(ShouldRestorePartitioningByLoad) { - ShouldRestoreSettings(R"( - partitioning_settings { - partitioning_by_load: ENABLED - } - )", { - NLs::PartitioningByLoadStatus(true), - }); - } - - Y_UNIT_TEST(ShouldRestoreMinMaxPartitionsCount) { - ShouldRestoreSettings(R"( - partitioning_settings { - min_partitions_count: 2 - max_partitions_count: 3 - } - )", { - NLs::MinPartitionsCountEqual(2), - NLs::MaxPartitionsCountEqual(3), - }); - } - - Y_UNIT_TEST(ShouldRestoreKeyBloomFilter) { - ShouldRestoreSettings(R"( - key_bloom_filter: ENABLED - )", { - NLs::KeyBloomFilterStatus(true), - }); - } - - Y_UNIT_TEST(ShouldRestorePerAzReadReplicas) { + const auto& config = desc.GetPathDescription().GetTable().GetPartitionConfig().GetColumnFamilies(0).GetStorageConfig(); + + UNIT_ASSERT_VALUES_EQUAL(config.GetSysLog().GetPreferredPoolKind(), "common"); + UNIT_ASSERT_VALUES_EQUAL(config.GetLog().GetPreferredPoolKind(), "common"); + UNIT_ASSERT_VALUES_EQUAL(config.GetExternal().GetPreferredPoolKind(), "external"); + UNIT_ASSERT(config.HasExternalThreshold()); + }; + + ShouldRestoreSettings(R"( + storage_settings { + tablet_commit_log0 { media: "common" } + tablet_commit_log1 { media: "common" } + external { media: "external" } + store_external_blobs: ENABLED + } + )", { + check, + }); + } + + Y_UNIT_TEST(ShouldRestoreColumnFamilies) { + ShouldRestoreSettings(R"( + storage_settings { + tablet_commit_log0 { media: "common" } + tablet_commit_log1 { media: "common" } + } + column_families { + name: "compressed" + data { media: "common" } + compression: COMPRESSION_LZ4 + } + )", { + NLs::ColumnFamiliesHas(1, "compressed"), + }); + } + + Y_UNIT_TEST(ShouldRestoreAttributes) { + ShouldRestoreSettings(R"( + attributes { + key: "key" + value: "value" + } + )", { + NLs::UserAttrsEqual({{"key", "value"}}), + }); + } + + // Skip compaction_policy (not supported) + // Skip uniform_partitions (has no effect) + + Y_UNIT_TEST(ShouldRestoreSplitPoints) { + ShouldRestoreSettings(R"( + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "b" } } + } + } + )", { + NLs::CheckBoundaries, + }); + } + + Y_UNIT_TEST(ShouldRestorePartitioningBySize) { + ShouldRestoreSettings(R"( + partitioning_settings { + partitioning_by_size: ENABLED + partition_size_mb: 1024 + } + )", { + NLs::SizeToSplitEqual(1 << 30), + }); + } + + Y_UNIT_TEST(ShouldRestorePartitioningByLoad) { + ShouldRestoreSettings(R"( + partitioning_settings { + partitioning_by_load: ENABLED + } + )", { + NLs::PartitioningByLoadStatus(true), + }); + } + + Y_UNIT_TEST(ShouldRestoreMinMaxPartitionsCount) { + ShouldRestoreSettings(R"( + partitioning_settings { + min_partitions_count: 2 + max_partitions_count: 3 + } + )", { + NLs::MinPartitionsCountEqual(2), + NLs::MaxPartitionsCountEqual(3), + }); + } + + Y_UNIT_TEST(ShouldRestoreKeyBloomFilter) { + ShouldRestoreSettings(R"( + key_bloom_filter: ENABLED + )", { + NLs::KeyBloomFilterStatus(true), + }); + } + + Y_UNIT_TEST(ShouldRestorePerAzReadReplicas) { NKikimrHive::TFollowerGroup group; group.SetFollowerCount(1); - group.SetRequireAllDataCenters(true); + group.SetRequireAllDataCenters(true); group.SetFollowerCountPerDataCenter(true); - - ShouldRestoreSettings(R"( - read_replicas_settings { - per_az_read_replicas_count: 1 - } - )", { + + ShouldRestoreSettings(R"( + read_replicas_settings { + per_az_read_replicas_count: 1 + } + )", { NLs::FollowerGroups({group}), - }); - } - - Y_UNIT_TEST(ShouldRestoreAnyAzReadReplicas) { + }); + } + + Y_UNIT_TEST(ShouldRestoreAnyAzReadReplicas) { NKikimrHive::TFollowerGroup group; group.SetFollowerCount(1); - group.SetRequireAllDataCenters(false); - - ShouldRestoreSettings(R"( - read_replicas_settings { - any_az_read_replicas_count: 1 - } - )", { + group.SetRequireAllDataCenters(false); + + ShouldRestoreSettings(R"( + read_replicas_settings { + any_az_read_replicas_count: 1 + } + )", { NLs::FollowerGroups({group}), - }); - } - - Y_UNIT_TEST(ShouldFailOnInvalidSchema) { - TTestBasicRuntime runtime; - - Run(runtime, ConvertTestData(GenerateTestData("", {{"a", 1}})), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", Ydb::StatusIds::CANCELLED); - } - - void ShouldFailOnInvalidCsv(const TString& csv) { - TTestBasicRuntime runtime; - - const auto data = TTestDataWithScheme(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {TTestData(csv, EmptyYsonStr)}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", Ydb::StatusIds::CANCELLED); - } - - Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { - ShouldFailOnInvalidCsv("\"a1\",\"value1\""); - } - - Y_UNIT_TEST(ShouldFailOnEmptyToken) { - ShouldFailOnInvalidCsv("\"a1\",\n"); - } - - Y_UNIT_TEST(ShouldFailOnInvalidValue) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UINT64 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", Ydb::StatusIds::CANCELLED); - } - - Y_UNIT_TEST(ShouldFailOnOutboundKey) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - partition_at_keys { - split_points { - type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } - value { items { text_value: "b" } } - } - } - )", {{"a", 1}, {"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", Ydb::StatusIds::CANCELLED); - } - - Y_UNIT_TEST(ShouldFailOnAbsentData) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - partition_at_keys { - split_points { - type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } - value { items { text_value: "b" } } - } - } - )", {{"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", Ydb::StatusIds::CANCELLED); - } - - void CancelShouldSucceed(TDelayFunc delayFunc) { - TTestBasicRuntime runtime; + }); + } + + Y_UNIT_TEST(ShouldFailOnInvalidSchema) { + TTestBasicRuntime runtime; + + Run(runtime, ConvertTestData(GenerateTestData("", {{"a", 1}})), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", Ydb::StatusIds::CANCELLED); + } + + void ShouldFailOnInvalidCsv(const TString& csv) { + TTestBasicRuntime runtime; + + const auto data = TTestDataWithScheme(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {TTestData(csv, EmptyYsonStr)}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", Ydb::StatusIds::CANCELLED); + } + + Y_UNIT_TEST(ShouldFailOnFileWithoutNewLines) { + ShouldFailOnInvalidCsv("\"a1\",\"value1\""); + } + + Y_UNIT_TEST(ShouldFailOnEmptyToken) { + ShouldFailOnInvalidCsv("\"a1\",\n"); + } + + Y_UNIT_TEST(ShouldFailOnInvalidValue) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UINT64 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", Ydb::StatusIds::CANCELLED); + } + + Y_UNIT_TEST(ShouldFailOnOutboundKey) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "b" } } + } + } + )", {{"a", 1}, {"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", Ydb::StatusIds::CANCELLED); + } + + Y_UNIT_TEST(ShouldFailOnAbsentData) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "b" } } + } + } + )", {{"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", Ydb::StatusIds::CANCELLED); + } + + void CancelShouldSucceed(TDelayFunc delayFunc) { + TTestBasicRuntime runtime; TTestEnv env(runtime, TTestEnvOptions()); - ui64 txId = 100; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - indexes { - name: "by_value" - index_columns: "value" - global_index {} - } - )", {{"a", 1}}); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - - THolder<IEventHandle> delayed; - auto prevObserver = SetDelayObserver(runtime, delayed, delayFunc); - - TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", port)); - const ui64 importId = txId; - - WaitForDelayed(runtime, delayed, prevObserver); - - TestCancelImport(runtime, ++txId, "/MyRoot", importId); - runtime.Send(delayed.Release(), 0, true); - env.TestWaitNotification(runtime, importId); - - TestGetImport(runtime, importId, "/MyRoot", Ydb::StatusIds::CANCELLED); - } - - Y_UNIT_TEST(CancelUponGettingSchemeShouldSucceed) { - CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { - return ev->GetTypeRewrite() == TEvPrivate::EvImportSchemeReady; - }); - } - - Y_UNIT_TEST(CancelUponCreatingTableShouldSucceed) { - CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { + ui64 txId = 100; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + indexes { + name: "by_value" + index_columns: "value" + global_index {} + } + )", {{"a", 1}}); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + + THolder<IEventHandle> delayed; + auto prevObserver = SetDelayObserver(runtime, delayed, delayFunc); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + const ui64 importId = txId; + + WaitForDelayed(runtime, delayed, prevObserver); + + TestCancelImport(runtime, ++txId, "/MyRoot", importId); + runtime.Send(delayed.Release(), 0, true); + env.TestWaitNotification(runtime, importId); + + TestGetImport(runtime, importId, "/MyRoot", Ydb::StatusIds::CANCELLED); + } + + Y_UNIT_TEST(CancelUponGettingSchemeShouldSucceed) { + CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { + return ev->GetTypeRewrite() == TEvPrivate::EvImportSchemeReady; + }); + } + + Y_UNIT_TEST(CancelUponCreatingTableShouldSucceed) { + CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) { - return false; - } - + return false; + } + return ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>()->Record .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpCreateTable; - }); - } - - Y_UNIT_TEST(CancelUponTransferringShouldSucceed) { - CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { + }); + } + + Y_UNIT_TEST(CancelUponTransferringShouldSucceed) { + CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) { - return false; - } - + return false; + } + return ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>()->Record .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpRestore; - }); - } - - Y_UNIT_TEST(CancelUponBuildingIndicesShouldSucceed) { - CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { + }); + } + + Y_UNIT_TEST(CancelUponBuildingIndicesShouldSucceed) { + CancelShouldSucceed([](TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) { - return false; - } - + return false; + } + return ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>()->Record .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpApplyIndexBuild; - }); - } -} - -Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { - void ShouldSucceed(const TString& scheme) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - const auto data = GenerateTestData(scheme, {{"a", 1}}); - - TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - } - - AsyncImport(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", port)); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - { - TInactiveZone inactive(activeZone); - TestGetImport(runtime, t.TxId, "/MyRoot", { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::NOT_FOUND - }); - } - }); - } - - Y_UNIT_TEST(ShouldSucceedOnSimpleTable) { - ShouldSucceed(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )"); - } - - Y_UNIT_TEST(ShouldSucceedOnIndexedTable) { - ShouldSucceed(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - indexes { - name: "by_value" - index_columns: "value" - global_index {} - } - )"); - } - - void CancelShouldSucceed(const TString& scheme) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - const auto data = GenerateTestData(scheme, {{"a", 1}}); - - TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - - runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - } - - AsyncImport(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", port)); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - const ui64 importId = t.TxId; - - t.TestEnv->ReliablePropose(runtime, CancelImportRequest(++t.TxId, "/MyRoot", importId), { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::NOT_FOUND - }); - t.TestEnv->TestWaitNotification(runtime, importId); - - { - TInactiveZone inactive(activeZone); - TestGetImport(runtime, importId, "/MyRoot", { - Ydb::StatusIds::SUCCESS, - Ydb::StatusIds::CANCELLED, - Ydb::StatusIds::NOT_FOUND - }); - } - }); - } - - Y_UNIT_TEST(CancelShouldSucceedOnSimpleTable) { - CancelShouldSucceed(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )"); - } - - Y_UNIT_TEST(CancelShouldSucceedOnIndexedTable) { - CancelShouldSucceed(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - indexes { - name: "by_value" - index_columns: "value" - global_index {} - } - )"); - } -} + }); + } +} + +Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { + void ShouldSucceed(const TString& scheme) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + const auto data = GenerateTestData(scheme, {{"a", 1}}); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + } + + AsyncImport(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestGetImport(runtime, t.TxId, "/MyRoot", { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::NOT_FOUND + }); + } + }); + } + + Y_UNIT_TEST(ShouldSucceedOnSimpleTable) { + ShouldSucceed(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )"); + } + + Y_UNIT_TEST(ShouldSucceedOnIndexedTable) { + ShouldSucceed(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + indexes { + name: "by_value" + index_columns: "value" + global_index {} + } + )"); + } + + void CancelShouldSucceed(const TString& scheme) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + const auto data = GenerateTestData(scheme, {{"a", 1}}); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + } + + AsyncImport(runtime, ++t.TxId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + const ui64 importId = t.TxId; + + t.TestEnv->ReliablePropose(runtime, CancelImportRequest(++t.TxId, "/MyRoot", importId), { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::NOT_FOUND + }); + t.TestEnv->TestWaitNotification(runtime, importId); + + { + TInactiveZone inactive(activeZone); + TestGetImport(runtime, importId, "/MyRoot", { + Ydb::StatusIds::SUCCESS, + Ydb::StatusIds::CANCELLED, + Ydb::StatusIds::NOT_FOUND + }); + } + }); + } + + Y_UNIT_TEST(CancelShouldSucceedOnSimpleTable) { + CancelShouldSucceed(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )"); + } + + Y_UNIT_TEST(CancelShouldSucceedOnIndexedTable) { + CancelShouldSucceed(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + indexes { + name: "by_value" + index_columns: "value" + global_index {} + } + )"); + } +} diff --git a/ydb/core/tx/schemeshard/ut_restore/ya.make b/ydb/core/tx/schemeshard/ut_restore/ya.make index b59c76a69ae..b1768fc897f 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ya.make +++ b/ydb/core/tx/schemeshard/ut_restore/ya.make @@ -1,36 +1,36 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(60) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( - contrib/libs/double-conversion - library/cpp/string_utils/quote + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + contrib/libs/double-conversion + library/cpp/string_utils/quote ydb/core/kqp/ut/common ydb/core/tx/schemeshard/ut_helpers ydb/core/wrappers/ut_helpers ydb/core/ydb_convert -) - -SRCS( - ut_restore.cpp -) - +) + +SRCS( + ut_restore.cpp +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/tx/schemeshard/ut_ru_calculator.cpp b/ydb/core/tx/schemeshard/ut_ru_calculator.cpp index 2823192816d..bb0e37459ac 100644 --- a/ydb/core/tx/schemeshard/ut_ru_calculator.cpp +++ b/ydb/core/tx/schemeshard/ut_ru_calculator.cpp @@ -1,48 +1,48 @@ -#include "schemeshard_billing_helpers.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/size_literals.h> - -namespace NKikimr { - -Y_UNIT_TEST_SUITE(TRUCalculatorTests) { +#include "schemeshard_billing_helpers.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/size_literals.h> + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(TRUCalculatorTests) { using TRUCalculator = NSchemeShard::TRUCalculator; - - Y_UNIT_TEST(TestReadTable) { - const ui64 costPerMB = 128; // 1 MB = 128 RU - - UNIT_ASSERT_VALUES_EQUAL(0 * costPerMB, TRUCalculator::ReadTable(0)); - UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1)); - - UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1_MB - 1)); - UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1_MB)); - UNIT_ASSERT_VALUES_EQUAL(2 * costPerMB, TRUCalculator::ReadTable(1_MB + 1)); - - UNIT_ASSERT_VALUES_EQUAL(2 * costPerMB, TRUCalculator::ReadTable(2_MB)); - UNIT_ASSERT_VALUES_EQUAL(10 * costPerMB, TRUCalculator::ReadTable(10_MB)); - } - - Y_UNIT_TEST(TestBulkUpsert) { - const ui64 x2CostPerKB = 1; // 1 KB = 0.5 RU - - UNIT_ASSERT_VALUES_EQUAL(0 * x2CostPerKB, TRUCalculator::BulkUpsert(0, 0)); // 0 RU - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(0, 1)); // 0.5 RU - - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB - 1, 1)); // 0.5 RU - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB - 1, 2)); // 1 RU - - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 1)); // 0.5 RU - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 2)); // 1 RU - - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 1)); // 1 RU - UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 2)); // 1 RU - UNIT_ASSERT_VALUES_EQUAL(2 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 3)); // 1.5 RU - - UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 10)); // 5 RU - UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(10_KB, 1)); // 5 RU - UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(10_KB, 10)); // 5 RU - } -} - -} // NKikimr + + Y_UNIT_TEST(TestReadTable) { + const ui64 costPerMB = 128; // 1 MB = 128 RU + + UNIT_ASSERT_VALUES_EQUAL(0 * costPerMB, TRUCalculator::ReadTable(0)); + UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1)); + + UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1_MB - 1)); + UNIT_ASSERT_VALUES_EQUAL(1 * costPerMB, TRUCalculator::ReadTable(1_MB)); + UNIT_ASSERT_VALUES_EQUAL(2 * costPerMB, TRUCalculator::ReadTable(1_MB + 1)); + + UNIT_ASSERT_VALUES_EQUAL(2 * costPerMB, TRUCalculator::ReadTable(2_MB)); + UNIT_ASSERT_VALUES_EQUAL(10 * costPerMB, TRUCalculator::ReadTable(10_MB)); + } + + Y_UNIT_TEST(TestBulkUpsert) { + const ui64 x2CostPerKB = 1; // 1 KB = 0.5 RU + + UNIT_ASSERT_VALUES_EQUAL(0 * x2CostPerKB, TRUCalculator::BulkUpsert(0, 0)); // 0 RU + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(0, 1)); // 0.5 RU + + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB - 1, 1)); // 0.5 RU + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB - 1, 2)); // 1 RU + + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 1)); // 0.5 RU + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 2)); // 1 RU + + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 1)); // 1 RU + UNIT_ASSERT_VALUES_EQUAL(1 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 2)); // 1 RU + UNIT_ASSERT_VALUES_EQUAL(2 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB + 1, 3)); // 1.5 RU + + UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(1_KB, 10)); // 5 RU + UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(10_KB, 1)); // 5 RU + UNIT_ASSERT_VALUES_EQUAL(5 * x2CostPerKB, TRUCalculator::BulkUpsert(10_KB, 10)); // 5 RU + } +} + +} // NKikimr diff --git a/ydb/core/tx/schemeshard/ut_ru_calculator/ya.make b/ydb/core/tx/schemeshard/ut_ru_calculator/ya.make index d24647b4344..013b3fb3c2e 100644 --- a/ydb/core/tx/schemeshard/ut_ru_calculator/ya.make +++ b/ydb/core/tx/schemeshard/ut_ru_calculator/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( - library/cpp/testing/unittest -) - -SRCS( - ut_ru_calculator.cpp -) - -END() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( + library/cpp/testing/unittest +) + +SRCS( + ut_ru_calculator.cpp +) + +END() diff --git a/ydb/core/tx/schemeshard/ut_subdomain.cpp b/ydb/core/tx/schemeshard/ut_subdomain.cpp index 7d49f853ee1..d0d30bf7da8 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain.cpp +++ b/ydb/core/tx/schemeshard/ut_subdomain.cpp @@ -110,7 +110,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), {NLs::PathExist}); - TestRmDir(runtime, txId++, "/MyRoot", "USER_0", {NKikimrScheme::StatusPathIsNotDirectory}); + TestRmDir(runtime, txId++, "/MyRoot", "USER_0", {NKikimrScheme::StatusPathIsNotDirectory}); } Y_UNIT_TEST(CreateAndWait) { @@ -353,7 +353,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/dir_0"), {NLs::InSubdomain, NLs::PathVersionEqual(5)}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/dir_0/table_1"), + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/dir_0/table_1"), {NLs::InSubdomain, NLs::PathVersionEqual(3)}); } @@ -814,7 +814,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { NLs::PathsInsideDomain(2), NLs::ShardsInsideDomain(0)}); - TestConsistentCopyTables(runtime, txId++, "/", R"( + TestConsistentCopyTables(runtime, txId++, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/USER_0/table" DstPath: "/MyRoot/USER_1/dst" @@ -824,7 +824,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { DstPath: "/MyRoot/USER_0/dst" })", {NKikimrScheme::StatusInvalidParameter}); - TestConsistentCopyTables(runtime, txId++, "/", R"( + TestConsistentCopyTables(runtime, txId++, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/USER_0/table" DstPath: "/MyRoot/USER_0/dst" @@ -835,7 +835,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { })", {NKikimrScheme::StatusInvalidParameter}); - TestConsistentCopyTables(runtime, txId++, "/", R"( + TestConsistentCopyTables(runtime, txId++, "/", R"( CopyTableDescriptions { SrcPath: "/MyRoot/USER_0/table" DstPath: "/MyRoot/USER_0/dst" @@ -1680,7 +1680,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { } - Y_UNIT_TEST(SimultaneousDefine) { + Y_UNIT_TEST(SimultaneousDefine) { TTestBasicRuntime runtime; TTestEnv env(runtime); ui64 txId = 100; @@ -1907,9 +1907,9 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { TTestEnv env(runtime); ui64 txId = 100; - TSchemeLimits lowLimits; - lowLimits.MaxPaths = 3; - lowLimits.MaxShards = 3; + TSchemeLimits lowLimits; + lowLimits.MaxPaths = 3; + lowLimits.MaxShards = 3; lowLimits.MaxPQPartitions = 300; SetSchemeshardSchemaLimits(runtime, lowLimits); @@ -1950,16 +1950,16 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { TTestEnv env(runtime); ui64 txId = 100; - TSchemeLimits lowLimits; - lowLimits.MaxDepth = 4; - lowLimits.MaxPaths = 3; - lowLimits.MaxChildrenInDir = 3; - lowLimits.MaxAclBytesSize = 25; - lowLimits.MaxTableColumns = 3; - lowLimits.MaxTableColumnNameLength = 10; - lowLimits.MaxTableKeyColumns = 1; + TSchemeLimits lowLimits; + lowLimits.MaxDepth = 4; + lowLimits.MaxPaths = 3; + lowLimits.MaxChildrenInDir = 3; + lowLimits.MaxAclBytesSize = 25; + lowLimits.MaxTableColumns = 3; + lowLimits.MaxTableColumnNameLength = 10; + lowLimits.MaxTableKeyColumns = 1; lowLimits.MaxShards = 6; - lowLimits.MaxShardsInPath = 4; + lowLimits.MaxShardsInPath = 4; lowLimits.MaxPQPartitions = 20; @@ -2103,7 +2103,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { UNIT_ASSERT_EQUAL(tableDest.size(), 1); dataShardId = tableDest.begin()->GetDatashardId(); }; - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1/2", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1/2", true), {NLs::PathExist, NLs::PathVersionEqual(3), NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), @@ -2184,129 +2184,129 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { NLs::ShardsInsideDomain(2)}); } - //create tables, column limits - { - TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); - env.TestWaitNotification(runtime, txId - 1); - - // MaxTableColumns - TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( - Name: "2" - Columns { Name: "RowId" Type: "Uint64" } - Columns { Name: "Value0" Type: "Utf8" } - Columns { Name: "Value1" Type: "Utf8" } - Columns { Name: "Value2" Type: "Utf8" } - KeyColumnNames: ["RowId"] + //create tables, column limits + { + TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); + env.TestWaitNotification(runtime, txId - 1); + + // MaxTableColumns + TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( + Name: "2" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "Value0" Type: "Utf8" } + Columns { Name: "Value1" Type: "Utf8" } + Columns { Name: "Value2" Type: "Utf8" } + KeyColumnNames: ["RowId"] )", {NKikimrScheme::StatusSchemeError}); - - // MaxTableColumnNameLength - TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( - Name: "3" - Columns { Name: "RowId" Type: "Uint64" } - Columns { Name: "VeryLongColumnName" Type: "Utf8" } - KeyColumnNames: ["RowId"] + + // MaxTableColumnNameLength + TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( + Name: "3" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "VeryLongColumnName" Type: "Utf8" } + KeyColumnNames: ["RowId"] )", {NKikimrScheme::StatusSchemeError}); - - // MaxTableKeyColumns - TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( - Name: "4" - Columns { Name: "RowId0" Type: "Uint64" } - Columns { Name: "RowId1" Type: "Uint64" } - Columns { Name: "Value" Type: "Utf8" } - KeyColumnNames: ["RowId0", "RowId1"] + + // MaxTableKeyColumns + TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( + Name: "4" + Columns { Name: "RowId0" Type: "Uint64" } + Columns { Name: "RowId1" Type: "Uint64" } + Columns { Name: "Value" Type: "Utf8" } + KeyColumnNames: ["RowId0", "RowId1"] )", {NKikimrScheme::StatusSchemeError}); - } - - //clear - { - auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); + } + + //clear + { + auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); TestForceDropUnsafe(runtime, txId++, dirVer.PathId.LocalPathId); - env.TestWaitNotification(runtime, txId - 1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), - {NLs::PathExist, + env.TestWaitNotification(runtime, txId - 1); + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), + {NLs::PathExist, NLs::PathVersionEqual(18), - NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), - NLs::PathsInsideDomain(0), + NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), + NLs::PathsInsideDomain(0), NLs::ShardsInsideDomain(2)}); - } - - //create dirs, acl size limit - { - TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); - env.TestWaitNotification(runtime, txId - 1); - - NACLib::TDiffACL tinyACL; - tinyACL.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericUse, "user0@builtin"); - TestModifyACL(runtime, txId++, "/MyRoot/USER_0", "1", tinyACL.SerializeAsString(), "user0@builtin"); - - NACLib::TDiffACL hugeACL; - for (ui32 i : xrange(100)) { - hugeACL.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericUse, Sprintf("user%" PRIu32 "@builtin", i)); - } - TestModifyACL(runtime, txId++, "/MyRoot/USER_0", "1", hugeACL.SerializeAsString(), "user0@builtin", + } + + //create dirs, acl size limit + { + TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); + env.TestWaitNotification(runtime, txId - 1); + + NACLib::TDiffACL tinyACL; + tinyACL.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericUse, "user0@builtin"); + TestModifyACL(runtime, txId++, "/MyRoot/USER_0", "1", tinyACL.SerializeAsString(), "user0@builtin"); + + NACLib::TDiffACL hugeACL; + for (ui32 i : xrange(100)) { + hugeACL.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericUse, Sprintf("user%" PRIu32 "@builtin", i)); + } + TestModifyACL(runtime, txId++, "/MyRoot/USER_0", "1", hugeACL.SerializeAsString(), "user0@builtin", NKikimrScheme::StatusInvalidParameter); - } - - //clear - { - auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); + } + + //clear + { + auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); TestForceDropUnsafe(runtime, txId++, dirVer.PathId.LocalPathId); - env.TestWaitNotification(runtime, txId - 1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), - {NLs::PathExist, + env.TestWaitNotification(runtime, txId - 1); + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), + {NLs::PathExist, NLs::PathVersionEqual(24), - NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), - NLs::PathsInsideDomain(0), + NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), + NLs::PathsInsideDomain(0), NLs::ShardsInsideDomain(2)}); - } - - //create tables, consistent copy targets limit - lowLimits.MaxPaths = 5; - lowLimits.MaxChildrenInDir = 4; - lowLimits.MaxConsistentCopyTargets = 1; - SetSchemeshardSchemaLimits(runtime, lowLimits); - { - TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); - TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( - Name: "2" - Columns { Name: "RowId" Type: "Uint64" } - Columns { Name: "Value" Type: "Utf8" } - KeyColumnNames: ["RowId"] - UniformPartitionsCount: 1 - )"); - TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( - Name: "3" - Columns { Name: "RowId" Type: "Uint64" } - Columns { Name: "Value" Type: "Utf8" } - KeyColumnNames: ["RowId"] - UniformPartitionsCount: 1 - )"); - env.TestWaitNotification(runtime, {txId - 1, txId - 2, txId - 3}); - - TestConsistentCopyTables(runtime, txId++, "/", R"( - CopyTableDescriptions { - SrcPath: "/MyRoot/USER_0/1/2" - DstPath: "/MyRoot/USER_0/1/12" - } - CopyTableDescriptions { - SrcPath: "/MyRoot/USER_0/1/3" - DstPath: "/MyRoot/USER_0/1/13" + } + + //create tables, consistent copy targets limit + lowLimits.MaxPaths = 5; + lowLimits.MaxChildrenInDir = 4; + lowLimits.MaxConsistentCopyTargets = 1; + SetSchemeshardSchemaLimits(runtime, lowLimits); + { + TestMkDir(runtime, txId++, "/MyRoot/USER_0", "1"); + TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( + Name: "2" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "Value" Type: "Utf8" } + KeyColumnNames: ["RowId"] + UniformPartitionsCount: 1 + )"); + TestCreateTable(runtime, txId++, "/MyRoot/USER_0/1", R"( + Name: "3" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "Value" Type: "Utf8" } + KeyColumnNames: ["RowId"] + UniformPartitionsCount: 1 + )"); + env.TestWaitNotification(runtime, {txId - 1, txId - 2, txId - 3}); + + TestConsistentCopyTables(runtime, txId++, "/", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/USER_0/1/2" + DstPath: "/MyRoot/USER_0/1/12" + } + CopyTableDescriptions { + SrcPath: "/MyRoot/USER_0/1/3" + DstPath: "/MyRoot/USER_0/1/13" })", {NKikimrScheme::StatusInvalidParameter}); - } - - //clear - { - auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); + } + + //clear + { + auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/1")); TestForceDropUnsafe(runtime, txId++, dirVer.PathId.LocalPathId); - env.TestWaitNotification(runtime, txId - 1); - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), - {NLs::PathExist, + env.TestWaitNotification(runtime, txId - 1); + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"), + {NLs::PathExist, NLs::PathVersionEqual(28), - NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), - NLs::PathsInsideDomain(0), + NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards), + NLs::PathsInsideDomain(0), NLs::ShardsInsideDomain(2)}); - } - + } + //databaseQuotas limits { @@ -2421,13 +2421,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { TTestEnv env(runtime); ui64 txId = 100; - TSchemeLimits lowLimits; - lowLimits.MaxDepth = 4; - lowLimits.MaxPaths = 5; - lowLimits.MaxChildrenInDir = 3; - lowLimits.MaxTableIndices = 4; + TSchemeLimits lowLimits; + lowLimits.MaxDepth = 4; + lowLimits.MaxPaths = 5; + lowLimits.MaxChildrenInDir = 3; + lowLimits.MaxTableIndices = 4; lowLimits.MaxShards = 7; - lowLimits.MaxShardsInPath = 4; + lowLimits.MaxShardsInPath = 4; lowLimits.ExtraPathSymbolsAllowed = "_.-"; SetSchemeshardSchemaLimits(runtime, lowLimits); @@ -2573,44 +2573,44 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { env.TestWaitNotification(runtime, {txId - 1, txId - 2, txId - 3, txId - 4, txId - 5, txId - 6}); } - - // MaxTableIndices - { - TestCreateIndexedTable(runtime, txId++, "/MyRoot/USER_0", R"( - TableDescription { - Name: "Table7" - Columns { Name: "RowId" Type: "Uint64" } - Columns { Name: "Value0" Type: "Utf8" } - Columns { Name: "Value1" Type: "Utf8" } - Columns { Name: "Value2" Type: "Utf8" } - Columns { Name: "Value3" Type: "Utf8" } - Columns { Name: "Value4" Type: "Utf8" } - KeyColumnNames: ["RowId"] - } - IndexDescription { - Name: "UserDefinedIndexByValue0" - KeyColumnNames: ["Value0"] - } - IndexDescription { - Name: "UserDefinedIndexByValue1" - KeyColumnNames: ["Value1"] - } - IndexDescription { - Name: "UserDefinedIndexByValue2" - KeyColumnNames: ["Value2"] - } - IndexDescription { - Name: "UserDefinedIndexByValue3" - KeyColumnNames: ["Value3"] - } - IndexDescription { - Name: "UserDefinedIndexByValue4" - KeyColumnNames: ["Value4"] - } + + // MaxTableIndices + { + TestCreateIndexedTable(runtime, txId++, "/MyRoot/USER_0", R"( + TableDescription { + Name: "Table7" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "Value0" Type: "Utf8" } + Columns { Name: "Value1" Type: "Utf8" } + Columns { Name: "Value2" Type: "Utf8" } + Columns { Name: "Value3" Type: "Utf8" } + Columns { Name: "Value4" Type: "Utf8" } + KeyColumnNames: ["RowId"] + } + IndexDescription { + Name: "UserDefinedIndexByValue0" + KeyColumnNames: ["Value0"] + } + IndexDescription { + Name: "UserDefinedIndexByValue1" + KeyColumnNames: ["Value1"] + } + IndexDescription { + Name: "UserDefinedIndexByValue2" + KeyColumnNames: ["Value2"] + } + IndexDescription { + Name: "UserDefinedIndexByValue3" + KeyColumnNames: ["Value3"] + } + IndexDescription { + Name: "UserDefinedIndexByValue4" + KeyColumnNames: ["Value4"] + } )", {NKikimrScheme::StatusResourceExhausted}); - - env.TestWaitNotification(runtime, txId - 1); - } + + env.TestWaitNotification(runtime, txId - 1); + } } Y_UNIT_TEST(SchemeLimitsCreatePq) { @@ -2806,95 +2806,95 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { KeyColumnNames: ["key"] )", {NKikimrScheme::StatusAccepted}); } - - Y_UNIT_TEST(DiskSpaceUsage) { - TTestBasicRuntime runtime; - TTestEnv env(runtime, TTestEnvOptions().EnablePersistentPartitionStats(true)); - const auto sender = runtime.AllocateEdgeActor(); - - auto writeRow = [&](ui64 tabletId, ui32 key, const TString& value, const char* table) { - NKikimrMiniKQL::TResult result; - TString error; + + Y_UNIT_TEST(DiskSpaceUsage) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnablePersistentPartitionStats(true)); + const auto sender = runtime.AllocateEdgeActor(); + + auto writeRow = [&](ui64 tabletId, ui32 key, const TString& value, const char* table) { + NKikimrMiniKQL::TResult result; + TString error; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( - ( - (let key '( '('key (Uint32 '%u ) ) ) ) - (let row '( '('value (Utf8 '%s) ) ) ) - (return (AsList (UpdateRow '__user__%s key row) )) - ) - )", key, value.c_str(), table), result, error); - + ( + (let key '( '('key (Uint32 '%u ) ) ) ) + (let row '( '('value (Utf8 '%s) ) ) ) + (return (AsList (UpdateRow '__user__%s key row) )) + ) + )", key, value.c_str(), table), result, error); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - }; - - auto waitForTableStats = [&](ui32 shards) { - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvDataShard::EvPeriodicTableStats, shards)); - runtime.DispatchEvents(options); - }; - - auto getDiskSpaceUsage = [&]() { - NKikimrSubDomains::TDiskSpaceUsage result; - - TestDescribeResult( - DescribePath(runtime, "/MyRoot"), { - NLs::PathExist, + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }; + + auto waitForTableStats = [&](ui32 shards) { + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvDataShard::EvPeriodicTableStats, shards)); + runtime.DispatchEvents(options); + }; + + auto getDiskSpaceUsage = [&]() { + NKikimrSubDomains::TDiskSpaceUsage result; + + TestDescribeResult( + DescribePath(runtime, "/MyRoot"), { + NLs::PathExist, NLs::Finished, [&result] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - result = record.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(); - } - } - ); - - return result; - }; - + result = record.GetPathDescription().GetDomainDescription().GetDiskSpaceUsage(); + } + } + ); + + return result; + }; + ui64 tabletId = TTestTxConfig::FakeHiveTablets; - ui64 txId = 100; - - // single-shard table - { - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table1" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "value" Type: "Utf8"} - KeyColumnNames: ["key"] + ui64 txId = 100; + + // single-shard table + { + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table1" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] )", {NKikimrScheme::StatusAccepted}); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId, 1, "value1", "Table1"); - waitForTableStats(1); - - auto du = getDiskSpaceUsage(); - UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); - + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId, 1, "value1", "Table1"); + waitForTableStats(1); + + auto du = getDiskSpaceUsage(); + UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); - UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); - } - - // multi-shard table - { - tabletId = tabletId + 1; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "Table2" - Columns { Name: "key" Type: "Uint32"} - Columns { Name: "value" Type: "Utf8"} - KeyColumnNames: ["key"] - UniformPartitionsCount: 2 + UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); + } + + // multi-shard table + { + tabletId = tabletId + 1; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table2" + Columns { Name: "key" Type: "Uint32"} + Columns { Name: "value" Type: "Utf8"} + KeyColumnNames: ["key"] + UniformPartitionsCount: 2 )", {NKikimrScheme::StatusAccepted}); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId + 0, 1, "value1", "Table2"); - writeRow(tabletId + 1, 2, "value2", "Table2"); - waitForTableStats(1 /* Table1 */ + 2 /* Table2 */); - - auto du = getDiskSpaceUsage(); - UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); - + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId + 0, 1, "value1", "Table2"); + writeRow(tabletId + 1, 2, "value2", "Table2"); + waitForTableStats(1 /* Table1 */ + 2 /* Table2 */); + + auto du = getDiskSpaceUsage(); + UNIT_ASSERT_C(du.GetTables().GetTotalSize() > 0, du.ShortDebugString()); + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); - UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); - } - } + UNIT_ASSERT_VALUES_EQUAL(du.ShortDebugString(), getDiskSpaceUsage().ShortDebugString()); + } + } Y_UNIT_TEST(DiskSpaceQuotas) { TTestBasicRuntime runtime; diff --git a/ydb/core/tx/schemeshard/ut_ttl.cpp b/ydb/core/tx/schemeshard/ut_ttl.cpp index 7f3232fe950..d512ad0033c 100644 --- a/ydb/core/tx/schemeshard/ut_ttl.cpp +++ b/ydb/core/tx/schemeshard/ut_ttl.cpp @@ -2,361 +2,361 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard_private.h> #include <ydb/core/tx/datashard/datashard.h> - -using namespace NKikimr; + +using namespace NKikimr; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - -static void CheckTTLSettings(TTestActorRuntime& runtime, const char* tableName = "TTLEnabledTable") { - TestDescribeResult( - DescribePath(runtime, Sprintf("/MyRoot/%s", tableName)), { - NLs::PathExist, +using namespace NSchemeShardUT_Private; + +static void CheckTTLSettings(TTestActorRuntime& runtime, const char* tableName = "TTLEnabledTable") { + TestDescribeResult( + DescribePath(runtime, Sprintf("/MyRoot/%s", tableName)), { + NLs::PathExist, NLs::Finished, [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& table = record.GetPathDescription().GetTable(); - UNIT_ASSERT(table.HasTTLSettings()); - - const auto& ttl = table.GetTTLSettings(); - UNIT_ASSERT(ttl.HasEnabled()); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "modified_at"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 3600); - } - } - ); -} - -Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { - void CreateTableShouldSucceed(const char* name, const char* ttlColumnType, const char* unit = "UNIT_AUTO") { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "%s" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "%s" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - ColumnUnit: %s - } - } - )", name, ttlColumnType, unit)); - env.TestWaitNotification(runtime, txId); - CheckTTLSettings(runtime, name); - } - - Y_UNIT_TEST(CreateTableShouldSucceed) { - for (auto ct : {"Date", "Datetime", "Timestamp"}) { - CreateTableShouldSucceed(Sprintf("TTLTableWith%sColumn", ct).data(), ct); - } - - for (auto ct : {"Uint32", "Uint64", "DyNumber"}) { - for (auto unit : {"UNIT_SECONDS", "UNIT_MILLISECONDS", "UNIT_MICROSECONDS", "UNIT_NANOSECONDS"}) { - CreateTableShouldSucceed(Sprintf("TTLTableWith%sColumn_%s", ct, unit).data(), ct, unit); - } - } - } - - Y_UNIT_TEST(CreateTableShouldFailOnUnknownColumn) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "created_at" - } - } + const auto& table = record.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasTTLSettings()); + + const auto& ttl = table.GetTTLSettings(); + UNIT_ASSERT(ttl.HasEnabled()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), "modified_at"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 3600); + } + } + ); +} + +Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { + void CreateTableShouldSucceed(const char* name, const char* ttlColumnType, const char* unit = "UNIT_AUTO") { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "%s" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "%s" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + ColumnUnit: %s + } + } + )", name, ttlColumnType, unit)); + env.TestWaitNotification(runtime, txId); + CheckTTLSettings(runtime, name); + } + + Y_UNIT_TEST(CreateTableShouldSucceed) { + for (auto ct : {"Date", "Datetime", "Timestamp"}) { + CreateTableShouldSucceed(Sprintf("TTLTableWith%sColumn", ct).data(), ct); + } + + for (auto ct : {"Uint32", "Uint64", "DyNumber"}) { + for (auto unit : {"UNIT_SECONDS", "UNIT_MILLISECONDS", "UNIT_MICROSECONDS", "UNIT_NANOSECONDS"}) { + CreateTableShouldSucceed(Sprintf("TTLTableWith%sColumn_%s", ct, unit).data(), ct, unit); + } + } + } + + Y_UNIT_TEST(CreateTableShouldFailOnUnknownColumn) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "created_at" + } + } )", {NKikimrScheme::StatusSchemeError}); - } - - Y_UNIT_TEST(CreateTableShouldFailOnWrongColumnType) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "String" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - } - } + } + + Y_UNIT_TEST(CreateTableShouldFailOnWrongColumnType) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "String" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + } + } )", {NKikimrScheme::StatusSchemeError}); - } - - void CreateTableShouldFailOnWrongUnit(const char* ttlColumnType, const char* unit) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "%s" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ColumnUnit: %s - } - } + } + + void CreateTableShouldFailOnWrongUnit(const char* ttlColumnType, const char* unit) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "%s" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ColumnUnit: %s + } + } )", ttlColumnType, unit), {NKikimrScheme::StatusSchemeError}); - } - - Y_UNIT_TEST(CreateTableShouldFailOnWrongUnit) { - for (auto ct : {"Date", "Datetime", "Timestamp"}) { - for (auto unit : {"UNIT_SECONDS", "UNIT_MILLISECONDS", "UNIT_MICROSECONDS", "UNIT_NANOSECONDS"}) { - CreateTableShouldFailOnWrongUnit(ct, unit); - } - } - - for (auto ct : {"Uint32", "Uint64", "DyNumber"}) { - CreateTableShouldFailOnWrongUnit(ct, "UNIT_AUTO"); - } - } - - Y_UNIT_TEST(CreateTableShouldFailOnUnspecifiedTTL) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - } + } + + Y_UNIT_TEST(CreateTableShouldFailOnWrongUnit) { + for (auto ct : {"Date", "Datetime", "Timestamp"}) { + for (auto unit : {"UNIT_SECONDS", "UNIT_MILLISECONDS", "UNIT_MICROSECONDS", "UNIT_NANOSECONDS"}) { + CreateTableShouldFailOnWrongUnit(ct, unit); + } + } + + for (auto ct : {"Uint32", "Uint64", "DyNumber"}) { + CreateTableShouldFailOnWrongUnit(ct, "UNIT_AUTO"); + } + } + + Y_UNIT_TEST(CreateTableShouldFailOnUnspecifiedTTL) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + } )", {NKikimrScheme::StatusSchemeError}); - } - - void CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { - const auto opts = TTestEnvOptions() - .EnableAsyncIndexes(true) - .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); - - TTestBasicRuntime runtime; - TTestEnv env(runtime, opts); - ui64 txId = 100; - - const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); - const auto status = shouldSucceed + } + + void CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { + const auto opts = TTestEnvOptions() + .EnableAsyncIndexes(true) + .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); + + TTestBasicRuntime runtime; + TTestEnv env(runtime, opts); + ui64 txId = 100; + + const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); + const auto status = shouldSucceed ? NKikimrScheme::StatusAccepted : NKikimrScheme::StatusPreconditionFailed; - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", Sprintf(R"( - TableDescription { - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - } - IndexDescription { - Name: "UserDefinedIndexByExpireAt" - KeyColumnNames: ["modified_at"] - Type: %s - } - )", NKikimrSchemeOp::EIndexType_Name(indexType).c_str()), {status}); - - if (shouldSucceed) { - env.TestWaitNotification(runtime, txId); - CheckTTLSettings(runtime); - } - } - - Y_UNIT_TEST(CreateTableShouldSucceedOnIndexedTable) { - CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobal); - } - - Y_UNIT_TEST(CreateTableShouldFailOnAsyncIndexedTable) { - CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); - } - - Y_UNIT_TEST(CreateTableShouldSucceedAsyncOnIndexedTable) { - CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); - } - - Y_UNIT_TEST(AlterTableShouldSuccess) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )"); - env.TestWaitNotification(runtime, txId); - CheckTTLSettings(runtime); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - DropColumns { Name: "modified_at" } + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + TableDescription { + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + } + IndexDescription { + Name: "UserDefinedIndexByExpireAt" + KeyColumnNames: ["modified_at"] + Type: %s + } + )", NKikimrSchemeOp::EIndexType_Name(indexType).c_str()), {status}); + + if (shouldSucceed) { + env.TestWaitNotification(runtime, txId); + CheckTTLSettings(runtime); + } + } + + Y_UNIT_TEST(CreateTableShouldSucceedOnIndexedTable) { + CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobal); + } + + Y_UNIT_TEST(CreateTableShouldFailOnAsyncIndexedTable) { + CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); + } + + Y_UNIT_TEST(CreateTableShouldSucceedAsyncOnIndexedTable) { + CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); + } + + Y_UNIT_TEST(AlterTableShouldSuccess) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + env.TestWaitNotification(runtime, txId); + CheckTTLSettings(runtime); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + DropColumns { Name: "modified_at" } )", {NKikimrScheme::StatusInvalidParameter}); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - TTLSettings { - Disabled { - } - } - )"); - env.TestWaitNotification(runtime, txId); - TestDescribeResult( - DescribePath(runtime, "/MyRoot/TTLEnabledTable"), { - NLs::PathExist, + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Disabled { + } + } + )"); + env.TestWaitNotification(runtime, txId); + TestDescribeResult( + DescribePath(runtime, "/MyRoot/TTLEnabledTable"), { + NLs::PathExist, NLs::Finished, [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - const auto& table = record.GetPathDescription().GetTable(); - UNIT_ASSERT(table.HasTTLSettings()); - UNIT_ASSERT(table.GetTTLSettings().HasDisabled()); - } - } - ); - } - - Y_UNIT_TEST(AlterTableShouldSuccessOnSimultaneousAddColumnAndEnableTTL) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "modified_at" Type: "Timestamp" } - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )"); - env.TestWaitNotification(runtime, txId); - CheckTTLSettings(runtime); - } - - Y_UNIT_TEST(AlterTableShouldFailOnSimultaneousDropColumnAndEnableTTL) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - DropColumns { Name: "modified_at" } - TTLSettings { - Enabled { - ColumnName: "modified_at" - } - } + const auto& table = record.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasTTLSettings()); + UNIT_ASSERT(table.GetTTLSettings().HasDisabled()); + } + } + ); + } + + Y_UNIT_TEST(AlterTableShouldSuccessOnSimultaneousAddColumnAndEnableTTL) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "modified_at" Type: "Timestamp" } + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + env.TestWaitNotification(runtime, txId); + CheckTTLSettings(runtime); + } + + Y_UNIT_TEST(AlterTableShouldFailOnSimultaneousDropColumnAndEnableTTL) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + DropColumns { Name: "modified_at" } + TTLSettings { + Enabled { + ColumnName: "modified_at" + } + } )", {NKikimrScheme::StatusInvalidParameter}); - } - - void AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { - const auto opts = TTestEnvOptions() - .EnableAsyncIndexes(true) - .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); - - TTestBasicRuntime runtime; - TTestEnv env(runtime, opts); - ui64 txId = 100; - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", Sprintf(R"( - TableDescription { - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - } - IndexDescription { - Name: "UserDefinedIndexByExpireAt" - KeyColumnNames: ["modified_at"] - Type: %s - } - )", NKikimrSchemeOp::EIndexType_Name(indexType).c_str())); - env.TestWaitNotification(runtime, txId); - - const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); - const auto status = shouldSucceed + } + + void AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { + const auto opts = TTestEnvOptions() + .EnableAsyncIndexes(true) + .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); + + TTestBasicRuntime runtime; + TTestEnv env(runtime, opts); + ui64 txId = 100; + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + TableDescription { + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndexByExpireAt" + KeyColumnNames: ["modified_at"] + Type: %s + } + )", NKikimrSchemeOp::EIndexType_Name(indexType).c_str())); + env.TestWaitNotification(runtime, txId); + + const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); + const auto status = shouldSucceed ? NKikimrScheme::StatusAccepted : NKikimrScheme::StatusPreconditionFailed; - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )", {status}); - - if (shouldSucceed) { - env.TestWaitNotification(runtime, txId); - CheckTTLSettings(runtime); - } - } - - Y_UNIT_TEST(AlterTableShouldSucceedOnIndexedTable) { - AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobal); - } - - Y_UNIT_TEST(AlterTableShouldFailOnAsyncIndexedTable) { - AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); - } - - Y_UNIT_TEST(AlterTableShouldSucceedOnAsyncIndexedTable) { - AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); - } - - void BuildIndex(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { - const auto opts = TTestEnvOptions() - .EnableAsyncIndexes(true) - .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); - + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )", {status}); + + if (shouldSucceed) { + env.TestWaitNotification(runtime, txId); + CheckTTLSettings(runtime); + } + } + + Y_UNIT_TEST(AlterTableShouldSucceedOnIndexedTable) { + AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobal); + } + + Y_UNIT_TEST(AlterTableShouldFailOnAsyncIndexedTable) { + AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); + } + + Y_UNIT_TEST(AlterTableShouldSucceedOnAsyncIndexedTable) { + AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); + } + + void BuildIndex(NKikimrSchemeOp::EIndexType indexType, bool enableTtlOnAsyncIndexedTables = false) { + const auto opts = TTestEnvOptions() + .EnableAsyncIndexes(true) + .EnableTtlOnAsyncIndexedTables(enableTtlOnAsyncIndexedTables); + TTestBasicRuntime runtime; - TTestEnv env(runtime, opts); + TTestEnv env(runtime, opts); ui64 txId = 100; TestCreateTable(runtime, ++txId, "/MyRoot", R"( @@ -375,469 +375,469 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { env.TestWaitNotification(runtime, txId); CheckTTLSettings(runtime); - const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); - const auto status = shouldSucceed - ? Ydb::StatusIds::SUCCESS - : Ydb::StatusIds::PRECONDITION_FAILED; + const bool shouldSucceed = (indexType != NKikimrSchemeOp::EIndexTypeGlobalAsync || enableTtlOnAsyncIndexedTables); + const auto status = shouldSucceed + ? Ydb::StatusIds::SUCCESS + : Ydb::StatusIds::PRECONDITION_FAILED; TestBuilIndex(runtime, ++txId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/TTLEnabledTable", - TBuildIndexConfig{"UserDefinedIndexByValue", indexType, {"value"}, {}}, status); - - if (shouldSucceed) { - env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/TTLEnabledTable"), { + TBuildIndexConfig{"UserDefinedIndexByValue", indexType, {"value"}, {}}, status); + + if (shouldSucceed) { + env.TestWaitNotification(runtime, txId); + TestDescribeResult(DescribePath(runtime, "/MyRoot/TTLEnabledTable"), { NLs::PathExist, NLs::Finished, - NLs::IndexesCount(1), - }); - } + NLs::IndexesCount(1), + }); + } + } + + Y_UNIT_TEST(BuildIndexShouldSucceed) { + BuildIndex(NKikimrSchemeOp::EIndexTypeGlobal); + } + + Y_UNIT_TEST(BuildAsyncIndexShouldFail) { + BuildIndex(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); + } + + Y_UNIT_TEST(BuildAsyncIndexShouldSucceed) { + BuildIndex(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); + } + + using TEvCondEraseReq = TEvDataShard::TEvConditionalEraseRowsRequest; + using TEvCondEraseResp = TEvDataShard::TEvConditionalEraseRowsResponse; + + void WaitForCondErase(TTestActorRuntimeBase& runtime, TEvCondEraseResp::ProtoRecordType::EStatus status = TEvCondEraseResp::ProtoRecordType::OK) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([status](IEventHandle& ev) -> bool { + if (ev.GetTypeRewrite() != TEvCondEraseResp::EventType) { + return false; + } + + auto resp = ev.Get<TEvCondEraseResp>(); + if (resp->Record.GetStatus() == TEvCondEraseResp::ProtoRecordType::ACCEPTED) { + return false; + } + + UNIT_ASSERT_VALUES_EQUAL_C(resp->Record.GetStatus(), status, resp->Record.GetErrorDescription()); + return true; + }); + + runtime.DispatchEvents(opts); } - - Y_UNIT_TEST(BuildIndexShouldSucceed) { - BuildIndex(NKikimrSchemeOp::EIndexTypeGlobal); - } - - Y_UNIT_TEST(BuildAsyncIndexShouldFail) { - BuildIndex(NKikimrSchemeOp::EIndexTypeGlobalAsync, false); - } - - Y_UNIT_TEST(BuildAsyncIndexShouldSucceed) { - BuildIndex(NKikimrSchemeOp::EIndexTypeGlobalAsync, true); - } - - using TEvCondEraseReq = TEvDataShard::TEvConditionalEraseRowsRequest; - using TEvCondEraseResp = TEvDataShard::TEvConditionalEraseRowsResponse; - - void WaitForCondErase(TTestActorRuntimeBase& runtime, TEvCondEraseResp::ProtoRecordType::EStatus status = TEvCondEraseResp::ProtoRecordType::OK) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([status](IEventHandle& ev) -> bool { - if (ev.GetTypeRewrite() != TEvCondEraseResp::EventType) { - return false; - } - - auto resp = ev.Get<TEvCondEraseResp>(); - if (resp->Record.GetStatus() == TEvCondEraseResp::ProtoRecordType::ACCEPTED) { - return false; - } - - UNIT_ASSERT_VALUES_EQUAL_C(resp->Record.GetStatus(), status, resp->Record.GetErrorDescription()); - return true; - }); - - runtime.DispatchEvents(opts); - } - - Y_UNIT_TEST(ConditionalErase) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - - auto writeRow = [&](ui64 tabletId, ui64 key, TInstant ts, const char* table, const char* ct = "Timestamp") { - NKikimrMiniKQL::TResult result; - TString error; + + Y_UNIT_TEST(ConditionalErase) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + + auto writeRow = [&](ui64 tabletId, ui64 key, TInstant ts, const char* table, const char* ct = "Timestamp") { + NKikimrMiniKQL::TResult result; + TString error; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( - ( - (let key '( '('key (Uint64 '%lu ) ) ) ) - (let row '( '('ts (%s '%lu) ) ) ) - (return (AsList (UpdateRow '__user__%s key row) )) - ) - )", key, ct, ts.GetValue(), table), result, error); - + ( + (let key '( '('key (Uint64 '%lu ) ) ) ) + (let row '( '('ts (%s '%lu) ) ) ) + (return (AsList (UpdateRow '__user__%s key row) )) + ) + )", key, ct, ts.GetValue(), table), result, error); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - }; - - auto readTable = [&](ui64 tabletId, const char* table) { - NKikimrMiniKQL::TResult result; - TString error; + UNIT_ASSERT_VALUES_EQUAL(error, ""); + }; + + auto readTable = [&](ui64 tabletId, const char* table) { + NKikimrMiniKQL::TResult result; + TString error; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, tabletId, Sprintf(R"( - ( - (let range '( '('key (Uint64 '0) (Void) ))) - (let columns '('key) ) - (let result (SelectRange '__user__%s range columns '())) - (return (AsList (SetResult 'Result result) )) - ) - )", table), result, error); - + ( + (let range '( '('key (Uint64 '0) (Void) ))) + (let columns '('key) ) + (let result (SelectRange '__user__%s range columns '())) + (return (AsList (SetResult 'Result result) )) + ) + )", table), result, error); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error); - UNIT_ASSERT_VALUES_EQUAL(error, ""); - - return result; - }; - - auto waitForScheduleCondErase = [&]() { - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvPrivate::EvRunConditionalErase)); - runtime.DispatchEvents(options); - }; - - auto setAllowConditionalEraseOperations = [&](bool value) { - TAtomic unused; - runtime.GetAppData().Icb->SetValue("SchemeShard_AllowConditionalEraseOperations", value, unused); - }; - - const TInstant now = TInstant::ParseIso8601("2020-09-18T18:00:00.000000Z"); - runtime.UpdateCurrentTime(now); - + UNIT_ASSERT_VALUES_EQUAL(error, ""); + + return result; + }; + + auto waitForScheduleCondErase = [&]() { + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvPrivate::EvRunConditionalErase)); + runtime.DispatchEvents(options); + }; + + auto setAllowConditionalEraseOperations = [&](bool value) { + TAtomic unused; + runtime.GetAppData().Icb->SetValue("SchemeShard_AllowConditionalEraseOperations", value, unused); + }; + + const TInstant now = TInstant::ParseIso8601("2020-09-18T18:00:00.000000Z"); + runtime.UpdateCurrentTime(now); + ui64 tabletId = TTestTxConfig::FakeHiveTablets; - ui64 txId = 100; - - { - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable1" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - } - } - )"); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId, 1, now, "TTLEnabledTable1"); - { - auto result = readTable(tabletId, "TTLEnabledTable1"); + ui64 txId = 100; + + { + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable1" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + } + } + )"); + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId, 1, now, "TTLEnabledTable1"); + { + auto result = readTable(tabletId, "TTLEnabledTable1"); NKqp::CompareYson(R"([[[[[["1"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Minutes(1)); - WaitForCondErase(runtime); - { - auto result = readTable(tabletId, "TTLEnabledTable1"); + } + + runtime.AdvanceCurrentTime(TDuration::Minutes(1)); + WaitForCondErase(runtime); + { + auto result = readTable(tabletId, "TTLEnabledTable1"); NKqp::CompareYson(R"([[[[];%false]]])", result); - } - } - - { - ++tabletId; - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable2" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - ExpireAfterSeconds: 3600 - } - } - )"); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId, 1, now - TDuration::Hours(1), "TTLEnabledTable2"); - writeRow(tabletId, 2, now, "TTLEnabledTable2"); - { - auto result = readTable(tabletId, "TTLEnabledTable2"); + } + } + + { + ++tabletId; + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable2" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + ExpireAfterSeconds: 3600 + } + } + )"); + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId, 1, now - TDuration::Hours(1), "TTLEnabledTable2"); + writeRow(tabletId, 2, now, "TTLEnabledTable2"); + { + auto result = readTable(tabletId, "TTLEnabledTable2"); NKqp::CompareYson(R"([[[[[["1"]];[["2"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Minutes(1)); - WaitForCondErase(runtime); - { - auto result = readTable(tabletId, "TTLEnabledTable2"); + } + + runtime.AdvanceCurrentTime(TDuration::Minutes(1)); + WaitForCondErase(runtime); + { + auto result = readTable(tabletId, "TTLEnabledTable2"); NKqp::CompareYson(R"([[[[[["2"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Hours(1)); - WaitForCondErase(runtime); - { - auto result = readTable(tabletId, "TTLEnabledTable2"); + } + + runtime.AdvanceCurrentTime(TDuration::Hours(1)); + WaitForCondErase(runtime); + { + auto result = readTable(tabletId, "TTLEnabledTable2"); NKqp::CompareYson(R"([[[[];%false]]])", result); - } - } - - { - setAllowConditionalEraseOperations(false); - - ++tabletId; - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable3" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - } - } - )"); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId, 1, now, "TTLEnabledTable3"); - { - auto result = readTable(tabletId, "TTLEnabledTable3"); + } + } + + { + setAllowConditionalEraseOperations(false); + + ++tabletId; + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable3" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + } + } + )"); + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId, 1, now, "TTLEnabledTable3"); + { + auto result = readTable(tabletId, "TTLEnabledTable3"); NKqp::CompareYson(R"([[[[[["1"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Minutes(1)); - waitForScheduleCondErase(); - { - auto result = readTable(tabletId, "TTLEnabledTable3"); + } + + runtime.AdvanceCurrentTime(TDuration::Minutes(1)); + waitForScheduleCondErase(); + { + auto result = readTable(tabletId, "TTLEnabledTable3"); NKqp::CompareYson(R"([[[[[["1"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Hours(1)); - waitForScheduleCondErase(); - { - auto result = readTable(tabletId, "TTLEnabledTable3"); + } + + runtime.AdvanceCurrentTime(TDuration::Hours(1)); + waitForScheduleCondErase(); + { + auto result = readTable(tabletId, "TTLEnabledTable3"); NKqp::CompareYson(R"([[[[[["1"]]];%false]]])", result); - } - } - - { - setAllowConditionalEraseOperations(true); - - ++tabletId; - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable4" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Uint64" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - ColumnUnit: UNIT_MICROSECONDS - } - } - )"); - env.TestWaitNotification(runtime, txId); - - writeRow(tabletId, 1, now, "TTLEnabledTable4", "Uint64"); - writeRow(tabletId, 2, now + TDuration::Days(1), "TTLEnabledTable4", "Uint64"); - { - auto result = readTable(tabletId, "TTLEnabledTable4"); + } + } + + { + setAllowConditionalEraseOperations(true); + + ++tabletId; + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable4" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Uint64" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + ColumnUnit: UNIT_MICROSECONDS + } + } + )"); + env.TestWaitNotification(runtime, txId); + + writeRow(tabletId, 1, now, "TTLEnabledTable4", "Uint64"); + writeRow(tabletId, 2, now + TDuration::Days(1), "TTLEnabledTable4", "Uint64"); + { + auto result = readTable(tabletId, "TTLEnabledTable4"); NKqp::CompareYson(R"([[[[[["1"]];[["2"]]];%false]]])", result); - } - - runtime.AdvanceCurrentTime(TDuration::Minutes(1)); - WaitForCondErase(runtime); - { - auto result = readTable(tabletId, "TTLEnabledTable4"); + } + + runtime.AdvanceCurrentTime(TDuration::Minutes(1)); + WaitForCondErase(runtime); + { + auto result = readTable(tabletId, "TTLEnabledTable4"); NKqp::CompareYson(R"([[[[[["2"]]];%false]]])", result); - } - } - } - - Y_UNIT_TEST(RacyAlterTableAndConditionalErase) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - auto delayConditionalErase = [&]() -> THolder<IEventHandle> { - THolder<IEventHandle> delayed; - - auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { - switch (ev->GetTypeRewrite()) { - case TEvCondEraseReq::EventType: - delayed.Reset(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - default: - return TTestActorRuntime::EEventAction::PROCESS; - } - }); - - if (!delayed) { - TDispatchOptions opts; - opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { - return bool(delayed); - }); - runtime.DispatchEvents(opts); - } - - runtime.SetObserverFunc(prevObserver); - return delayed; - }; - - TestCreateTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - } - } - )"); - env.TestWaitNotification(runtime, txId); - - auto delayed = delayConditionalErase(); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "value" Type: "String" } - )"); - env.TestWaitNotification(runtime, txId); - - runtime.Send(delayed.Release(), 0, true); - WaitForCondErase(runtime, TEvCondEraseResp::ProtoRecordType::SCHEME_ERROR); - } - - Y_UNIT_TEST(ShouldCheckQuotas) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateSubDomain(runtime, ++txId, "/MyRoot", R"( - Name: "SubDomain" - PlanResolution: 50 - Coordinators: 1 - Mediators: 1 - TimeCastBucketsPerMediator: 2 - DatabaseQuotas { - ttl_min_run_internal_seconds: 1800 - } - )"); - env.TestWaitNotification(runtime, txId); - - // ok without sys settings - TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( - Name: "Table1" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )"); - env.TestWaitNotification(runtime, txId); - - // ok (RunInterval >= limit) - TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( - Name: "Table3" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - SysSettings { - RunInterval: 1800000000 - } - } - } - )"); - env.TestWaitNotification(runtime, txId); - - // error (RunInterval < limit) - TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( - Name: "Table4" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - SysSettings { - RunInterval: 1799999999 - } - } - } + } + } + } + + Y_UNIT_TEST(RacyAlterTableAndConditionalErase) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + auto delayConditionalErase = [&]() -> THolder<IEventHandle> { + THolder<IEventHandle> delayed; + + auto prevObserver = runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + switch (ev->GetTypeRewrite()) { + case TEvCondEraseReq::EventType: + delayed.Reset(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + default: + return TTestActorRuntime::EEventAction::PROCESS; + } + }); + + if (!delayed) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool { + return bool(delayed); + }); + runtime.DispatchEvents(opts); + } + + runtime.SetObserverFunc(prevObserver); + return delayed; + }; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + } + } + )"); + env.TestWaitNotification(runtime, txId); + + auto delayed = delayConditionalErase(); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "value" Type: "String" } + )"); + env.TestWaitNotification(runtime, txId); + + runtime.Send(delayed.Release(), 0, true); + WaitForCondErase(runtime, TEvCondEraseResp::ProtoRecordType::SCHEME_ERROR); + } + + Y_UNIT_TEST(ShouldCheckQuotas) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateSubDomain(runtime, ++txId, "/MyRoot", R"( + Name: "SubDomain" + PlanResolution: 50 + Coordinators: 1 + Mediators: 1 + TimeCastBucketsPerMediator: 2 + DatabaseQuotas { + ttl_min_run_internal_seconds: 1800 + } + )"); + env.TestWaitNotification(runtime, txId); + + // ok without sys settings + TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( + Name: "Table1" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + env.TestWaitNotification(runtime, txId); + + // ok (RunInterval >= limit) + TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( + Name: "Table3" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + SysSettings { + RunInterval: 1800000000 + } + } + } + )"); + env.TestWaitNotification(runtime, txId); + + // error (RunInterval < limit) + TestCreateTable(runtime, ++txId, "/MyRoot/SubDomain", R"( + Name: "Table4" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + SysSettings { + RunInterval: 1799999999 + } + } + } )", {NKikimrScheme::StatusSchemeError}); - } - - Y_UNIT_TEST(ShouldSkipDroppedColumn) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( - TableDescription { - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "ts" Type: "Timestamp" } - Columns { Name: "indexed" Type: "Uint64" } - Columns { Name: "extra" Type: "Uint64" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "ts" - } - } - } - IndexDescription { - Name: "Index" - KeyColumnNames: ["indexed"] - } - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - DropColumns { Name: "extra" } - )"); - env.TestWaitNotification(runtime, txId); - - TestAlterTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "extra" Type: "Uint64" } - )"); - env.TestWaitNotification(runtime, txId); - - WaitForCondErase(runtime); - } -} - -Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { - Y_UNIT_TEST(CreateTable) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - { - TInactiveZone inactive(activeZone); - CheckTTLSettings(runtime); - } - }); - } - - Y_UNIT_TEST(AlterTable) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "TTLEnabledTable" - Columns { Name: "key" Type: "Uint64" } - Columns { Name: "modified_at" Type: "Timestamp" } - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - } - - TestAlterTable(runtime, ++t.TxId, "/MyRoot", R"( - Name: "TTLEnabledTable" - TTLSettings { - Enabled { - ColumnName: "modified_at" - ExpireAfterSeconds: 3600 - } - } - )"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); - - { - TInactiveZone inactive(activeZone); - CheckTTLSettings(runtime); - } - }); - } -} + } + + Y_UNIT_TEST(ShouldSkipDroppedColumn) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "ts" Type: "Timestamp" } + Columns { Name: "indexed" Type: "Uint64" } + Columns { Name: "extra" Type: "Uint64" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "ts" + } + } + } + IndexDescription { + Name: "Index" + KeyColumnNames: ["indexed"] + } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + DropColumns { Name: "extra" } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "extra" Type: "Uint64" } + )"); + env.TestWaitNotification(runtime, txId); + + WaitForCondErase(runtime); + } +} + +Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { + Y_UNIT_TEST(CreateTable) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + CheckTTLSettings(runtime); + } + }); + } + + Y_UNIT_TEST(AlterTable) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestAlterTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + CheckTTLSettings(runtime); + } + }); + } +} diff --git a/ydb/core/tx/schemeshard/ut_ttl/ya.make b/ydb/core/tx/schemeshard/ut_ttl/ya.make index 9e72b55c993..2fe57635d80 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/ya.make +++ b/ydb/core/tx/schemeshard/ut_ttl/ya.make @@ -1,32 +1,32 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -SPLIT_FACTOR(60) - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - TIMEOUT(3600) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -PEERDIR( + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( ydb/core/kqp/ut/common ydb/core/tx/schemeshard/ut_helpers -) - -SRCS( - ut_ttl.cpp -) - +) + +SRCS( + ut_ttl.cpp +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/tx/schemeshard/ut_upgradesubdomain.cpp b/ydb/core/tx/schemeshard/ut_upgradesubdomain.cpp index 10871258919..1ee6456acb1 100644 --- a/ydb/core/tx/schemeshard/ut_upgradesubdomain.cpp +++ b/ydb/core/tx/schemeshard/ut_upgradesubdomain.cpp @@ -1,7 +1,7 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> -#include <util/system/env.h> - +#include <util/system/env.h> + using namespace NKikimr; using namespace NSchemeShard; using namespace NSchemeShardUT_Private; diff --git a/ydb/core/tx/schemeshard/ut_upgradesubdomain/ya.make b/ydb/core/tx/schemeshard/ut_upgradesubdomain/ya.make index a079e0d9ed2..7759750ff30 100644 --- a/ydb/core/tx/schemeshard/ut_upgradesubdomain/ya.make +++ b/ydb/core/tx/schemeshard/ut_upgradesubdomain/ya.make @@ -6,7 +6,7 @@ IF (NOT WITH_VALGRIND) TIMEOUT(600) SIZE(MEDIUM) - + INCLUDE(${ARCADIA_ROOT}/ydb/tests/supp/ubsan_supp.inc) PEERDIR( diff --git a/ydb/core/tx/schemeshard/ut_upgradesubdomain_reboots.cpp b/ydb/core/tx/schemeshard/ut_upgradesubdomain_reboots.cpp index 809360e5f4c..d1a4a4020b0 100644 --- a/ydb/core/tx/schemeshard/ut_upgradesubdomain_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_upgradesubdomain_reboots.cpp @@ -1,7 +1,7 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> -#include <util/system/env.h> - +#include <util/system/env.h> + using namespace NKikimr; using namespace NSchemeShard; using namespace NSchemeShardUT_Private; @@ -181,9 +181,9 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainWithOutDesicion) { Columns { Name: "addColumn2" Type: "Uint64"} )", {TEvSchemeShard::EStatus::StatusMultipleModifications}); - TestUserAttrs(runtime, ++t.TxId, "/MyRoot/USER_0", "DirA", - {TEvSchemeShard::EStatus::StatusMultipleModifications}, - AlterUserAttrs({{"AttrA2", "ValA2"}})); + TestUserAttrs(runtime, ++t.TxId, "/MyRoot/USER_0", "DirA", + {TEvSchemeShard::EStatus::StatusMultipleModifications}, + AlterUserAttrs({{"AttrA2", "ValA2"}})); t.TestEnv->TestWaitNotification(runtime, {t.TxId-2, t.TxId-1, t.TxId}); @@ -251,9 +251,9 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainWithOutDesicion) { Columns { Name: "addColumn2" Type: "Uint64"} )", {TEvSchemeShard::EStatus::StatusMultipleModifications}); - TestUserAttrs(runtime, ++t.TxId, "/MyRoot/USER_0", "DirA", - {TEvSchemeShard::EStatus::StatusMultipleModifications}, - AlterUserAttrs({{"AttrA2", "ValA2"}})); + TestUserAttrs(runtime, ++t.TxId, "/MyRoot/USER_0", "DirA", + {TEvSchemeShard::EStatus::StatusMultipleModifications}, + AlterUserAttrs({{"AttrA2", "ValA2"}})); t.TestEnv->TestWaitNotification(runtime, {t.TxId-2, t.TxId-1, t.TxId}); @@ -421,7 +421,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainUndo) { } } - + Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { Y_UNIT_TEST(Kesus) { TTestWithReboots t; @@ -854,8 +854,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { NLs::IsExternalSubDomain("USER_0"), NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - TestUserAttrs(runtime, tenantSchemeShard, ++t.TxId, "/MyRoot/USER_0", "DirA", - {NKikimrScheme::StatusAccepted}, AlterUserAttrs(attrs2, {"Name1"})); + TestUserAttrs(runtime, tenantSchemeShard, ++t.TxId, "/MyRoot/USER_0", "DirA", + {NKikimrScheme::StatusAccepted}, AlterUserAttrs(attrs2, {"Name1"})); t.TestEnv->TestWaitNotification(runtime, t.TxId, tenantSchemeShard); TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA"), @@ -911,7 +911,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { NLs::IsExternalSubDomain("USER_0"), NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(1)}); @@ -925,7 +925,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { )"); t.TestEnv->TestWaitNotification(runtime, t.TxId, tenantSchemeShard); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(2)}); @@ -986,7 +986,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { NLs::IsExternalSubDomain("USER_0"), NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(1)}); @@ -1042,14 +1042,14 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { NLs::IsExternalSubDomain("USER_0"), NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(1)}); TestCopyTable(runtime, tenantSchemeShard, ++t.TxId, "/MyRoot/USER_0/DirA", "Copy", "/MyRoot/USER_0/DirA/Table"); t.TestEnv->TestWaitNotification(runtime, t.TxId, tenantSchemeShard); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Copy", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Copy", true), {NLs::PathExist, NLs::PartitionCount(1)}); }); @@ -1087,7 +1087,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { )"); t.TestEnv->TestWaitNotification(runtime, {t.TxId - 1, t.TxId}); - TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(1), NLs::IndexesCount(1)}); @@ -1104,7 +1104,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardUpgradeSubDomainCommit) { {NLs::PathRedirected, NLs::ExtractTenantSchemeshard(&tenantSchemeShard)}); - TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), + TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/DirA/Table", true), {NLs::PathExist, NLs::PartitionCount(1), NLs::IndexesCount(1)}); diff --git a/ydb/core/tx/schemeshard/ut_user_attributes.cpp b/ydb/core/tx/schemeshard/ut_user_attributes.cpp index 735c94b0926..48a63f877e3 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes.cpp +++ b/ydb/core/tx/schemeshard/ut_user_attributes.cpp @@ -77,21 +77,21 @@ Y_UNIT_TEST_SUITE(TSchemeShardUserAttrsTest) { auto dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::UserAttrsEqual({{"AttrA1", "ValA1"}}), NLs::PathVersionEqual(3)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, - AlterUserAttrs({}, {"AttrA1"}), {dirVer}); + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, + AlterUserAttrs({}, {"AttrA1"}), {dirVer}); env.TestWaitNotification(runtime, txId-1); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::UserAttrsEqual({}), NLs::PathVersionEqual(4)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusPreconditionFailed}, - AlterUserAttrs({{"AttrA2", "ValA2"}}), {dirVer}); + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusPreconditionFailed}, + AlterUserAttrs({{"AttrA2", "ValA2"}}), {dirVer}); env.TestWaitNotification(runtime, txId-1); dirVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::UserAttrsEqual({}), NLs::PathVersionEqual(4)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, - AlterUserAttrs({{"AttrA2", "ValA2"}}), {dirVer}); + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, + AlterUserAttrs({{"AttrA2", "ValA2"}}), {dirVer}); env.TestWaitNotification(runtime, txId-1); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::UserAttrsEqual({{"AttrA2", "ValA2"}}), NLs::PathVersionEqual(5)}); @@ -130,10 +130,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardUserAttrsTest) { auto dirCVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirC"), {NLs::Finished, NLs::PathVersionEqual(3)}); - TestRmDir(runtime, txId++, "/MyRoot", "DirC", {NKikimrScheme::StatusPreconditionFailed}, {dirCVer, dirBVer, dirAVer, rootVer}); + TestRmDir(runtime, txId++, "/MyRoot", "DirC", {NKikimrScheme::StatusPreconditionFailed}, {dirCVer, dirBVer, dirAVer, rootVer}); rootVer = TestDescribeResult(DescribePath(runtime, "/MyRoot"), {NLs::Finished}); - TestRmDir(runtime, txId++, "/MyRoot", "DirC", {NKikimrScheme::StatusAccepted}, {dirCVer, dirBVer, dirAVer, rootVer}); + TestRmDir(runtime, txId++, "/MyRoot", "DirC", {NKikimrScheme::StatusAccepted}, {dirCVer, dirBVer, dirAVer, rootVer}); env.TestWaitNotification(runtime, txId-1); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirC"), {NLs::PathNotExist}); @@ -170,20 +170,20 @@ Y_UNIT_TEST_SUITE(TSchemeShardUserAttrsTest) { auto dirAVer =TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::UserAttrsEqual({}), NLs::PathVersionEqual(6)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, AlterUserAttrs({{"AttrA6", "ValA6"}}), {dirAVer}); env.TestWaitNotification(runtime, txId-1); TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::Finished, NLs::UserAttrsEqual({{"AttrA6", "ValA6"}}), NLs::PathVersionEqual(7)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusPreconditionFailed}, - AlterUserAttrs({}, {"AttrA6"}), {dirAVer}); + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusPreconditionFailed}, + AlterUserAttrs({}, {"AttrA6"}), {dirAVer}); dirAVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), {NLs::Finished, NLs::UserAttrsEqual({{"AttrA6", "ValA6"}}), NLs::PathVersionEqual(7)}); - TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, + TestUserAttrs(runtime, txId++, "/MyRoot", "DirA", {NKikimrScheme::StatusAccepted}, AlterUserAttrs({}, {"AttrA6"}), {dirAVer}); env.TestWaitNotification(runtime, txId-1); @@ -207,15 +207,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardUserAttrsTest) { auto dirBBVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/DirBB"), {NLs::Finished, NLs::PathVersionEqual(3)}); - TestRmDir(runtime, txId++, "/MyRoot/DirB", "DirBB", {NKikimrScheme::StatusPreconditionFailed}, {dirAVer, dirBVer}); + TestRmDir(runtime, txId++, "/MyRoot/DirB", "DirBB", {NKikimrScheme::StatusPreconditionFailed}, {dirAVer, dirBVer}); dirBVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), {NLs::Finished, NLs::PathVersionEqual(5)}); - TestRmDir(runtime, txId++, "/MyRoot/DirB", "DirBB", {NKikimrScheme::StatusAccepted}, {dirAVer, dirBVer}); + TestRmDir(runtime, txId++, "/MyRoot/DirB", "DirBB", {NKikimrScheme::StatusAccepted}, {dirAVer, dirBVer}); env.TestWaitNotification(runtime, txId-1); dirBVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), {NLs::Finished, NLs::NoChildren, NLs::PathVersionEqual(7)}); - TestRmDir(runtime, txId++, "/MyRoot", "DirB", {NKikimrScheme::StatusPreconditionFailed}, {dirAVer, dirBVer, dirBBVer}); + TestRmDir(runtime, txId++, "/MyRoot", "DirB", {NKikimrScheme::StatusPreconditionFailed}, {dirAVer, dirBVer, dirBBVer}); } Y_UNIT_TEST(SpecialAttributes) { diff --git a/ydb/core/tx/schemeshard/ut_user_attributes_reboots.cpp b/ydb/core/tx/schemeshard/ut_user_attributes_reboots.cpp index 5555bb534f0..8b58b94b0d7 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_user_attributes_reboots.cpp @@ -36,8 +36,8 @@ Y_UNIT_TEST_SUITE(TUserAttrsTestWithReboots) { NLs::UserAttrsEqual(userAttrs)}); } - t.TestEnv->ReliablePropose(runtime, UserAttrsRequest(++t.TxId, "/MyRoot/DirA", "USER_0", - AlterUserAttrs({}, userAttrsKeys), {pathVer}), + t.TestEnv->ReliablePropose(runtime, UserAttrsRequest(++t.TxId, "/MyRoot/DirA", "USER_0", + AlterUserAttrs({}, userAttrsKeys), {pathVer}), {NKikimrScheme::StatusAccepted, NKikimrScheme::StatusMultipleModifications, NKikimrScheme::StatusPreconditionFailed}); t.TestEnv->TestWaitNotification(runtime, t.TxId); diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 6808a11de96..ceb87ce3883 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -5,10 +5,10 @@ RECURSE_FOR_TESTS( ut_bsvolume ut_bsvolume_reboots ut_cdc_stream - ut_cdc_stream_reboots + ut_cdc_stream_reboots ut_compaction ut_export - ut_export_reboots_s3 + ut_export_reboots_s3 ut_extsubdomain ut_extsubdomain_reboots ut_filestore_reboots @@ -22,8 +22,8 @@ RECURSE_FOR_TESTS( ut_pq ut_pq_reboots ut_reboots - ut_replication - ut_replication_reboots + ut_replication + ut_replication_reboots ut_restore ut_rtmr ut_rtmr_reboots @@ -61,13 +61,13 @@ SRCS( schemeshard.cpp schemeshard__compaction.cpp schemeshard__clean_pathes.cpp - schemeshard__conditional_erase.cpp + schemeshard__conditional_erase.cpp schemeshard__describe_scheme.cpp schemeshard__delete_tablet_reply.cpp schemeshard__find_subdomain_path_id.cpp - schemeshard__fix_bad_paths.cpp + schemeshard__fix_bad_paths.cpp schemeshard__init.cpp - schemeshard__init_populator.cpp + schemeshard__init_populator.cpp schemeshard__init_root.cpp schemeshard__init_schema.cpp schemeshard__serverless_storage_billing.cpp @@ -95,7 +95,7 @@ SRCS( schemeshard__operation_alter_table.cpp schemeshard__operation_alter_user_attrs.cpp schemeshard__operation_assign_bsv.cpp - schemeshard__operation_cancel_tx.cpp + schemeshard__operation_cancel_tx.cpp schemeshard__operation_common.cpp schemeshard__operation_common.h schemeshard__operation_consistent_copy_tables.cpp @@ -105,14 +105,14 @@ SRCS( schemeshard__operation_create_extsubdomain.cpp schemeshard__operation_create_fs.cpp schemeshard__operation_create_index.cpp - schemeshard__operation_create_indexed_table.cpp + schemeshard__operation_create_indexed_table.cpp schemeshard__operation_create_kesus.cpp schemeshard__operation_create_lock.cpp schemeshard__operation_create_olap_store.cpp schemeshard__operation_create_olap_table.cpp schemeshard__operation_create_pq.cpp - schemeshard__operation_create_replication.cpp - schemeshard__operation_create_restore.cpp + schemeshard__operation_create_replication.cpp + schemeshard__operation_create_restore.cpp schemeshard__operation_create_rtmr.cpp schemeshard__operation_create_sequence.cpp schemeshard__operation_create_solomon.cpp @@ -126,8 +126,8 @@ SRCS( schemeshard__operation_drop_lock.cpp schemeshard__operation_drop_olap_store.cpp schemeshard__operation_drop_olap_table.cpp - schemeshard__operation_drop_pq.cpp - schemeshard__operation_drop_replication.cpp + schemeshard__operation_drop_pq.cpp + schemeshard__operation_drop_replication.cpp schemeshard__operation_drop_sequence.cpp schemeshard__operation_drop_solomon.cpp schemeshard__operation_drop_subdomain.cpp @@ -150,10 +150,10 @@ SRCS( schemeshard__operation_apply_build_index.cpp schemeshard__operation_cansel_build_index.cpp schemeshard__operation_drop_index.cpp - schemeshard__operation_create_cdc_stream.cpp - schemeshard__operation_alter_cdc_stream.cpp - schemeshard__operation_drop_cdc_stream.cpp - schemeshard__publish_to_scheme_board.cpp + schemeshard__operation_create_cdc_stream.cpp + schemeshard__operation_alter_cdc_stream.cpp + schemeshard__operation_drop_cdc_stream.cpp + schemeshard__publish_to_scheme_board.cpp schemeshard__state_changed_reply.cpp schemeshard__table_stats.cpp schemeshard__table_stats_histogram.cpp @@ -162,7 +162,7 @@ SRCS( schemeshard__make_access_database_no_inheritable.cpp schemeshard_impl.cpp schemeshard_impl.h - schemeshard_billing_helpers.cpp + schemeshard_billing_helpers.cpp schemeshard_domain_links.h schemeshard_domain_links.cpp schemeshard_effective_acl.h @@ -170,7 +170,7 @@ SRCS( schemeshard_identificators.cpp schemeshard_info_types.cpp schemeshard_info_types.h - schemeshard_path_describer.cpp + schemeshard_path_describer.cpp schemeshard_path_element.h schemeshard_path.cpp schemeshard_path.h @@ -182,20 +182,20 @@ SRCS( schemeshard_user_attr_limits.h schemeshard_utils.cpp schemeshard_utils.h - schemeshard_export__cancel.cpp - schemeshard_export__create.cpp - schemeshard_export__forget.cpp - schemeshard_export__get.cpp - schemeshard_export__list.cpp - schemeshard_export_flow_proposals.cpp - schemeshard_export.cpp - schemeshard_import__cancel.cpp - schemeshard_import__create.cpp - schemeshard_import__forget.cpp - schemeshard_import__get.cpp - schemeshard_import__list.cpp - schemeshard_import_flow_proposals.cpp - schemeshard_import.cpp + schemeshard_export__cancel.cpp + schemeshard_export__create.cpp + schemeshard_export__forget.cpp + schemeshard_export__get.cpp + schemeshard_export__list.cpp + schemeshard_export_flow_proposals.cpp + schemeshard_export.cpp + schemeshard_import__cancel.cpp + schemeshard_import__create.cpp + schemeshard_import__forget.cpp + schemeshard_import__get.cpp + schemeshard_import__list.cpp + schemeshard_import_flow_proposals.cpp + schemeshard_import.cpp schemeshard_build_index.cpp schemeshard_build_index__cancel.cpp schemeshard_build_index__forget.cpp @@ -207,10 +207,10 @@ SRCS( operation_queue_timer.h ) -GENERATE_ENUM_SERIALIZATION(schemeshard_info_types.h) +GENERATE_ENUM_SERIALIZATION(schemeshard_info_types.h) GENERATE_ENUM_SERIALIZATION(schemeshard_types.h) - + PEERDIR( contrib/libs/protobuf library/cpp/deprecated/enum_codegen @@ -248,13 +248,13 @@ PEERDIR( YQL_LAST_ABI_VERSION() IF (OS_WINDOWS) - SRCS( - schemeshard_import_scheme_getter_fallback.cpp - ) -ELSE() - SRCS( - schemeshard_import_scheme_getter.cpp - ) -ENDIF() - + SRCS( + schemeshard_import_scheme_getter_fallback.cpp + ) +ELSE() + SRCS( + schemeshard_import_scheme_getter.cpp + ) +ENDIF() + END() diff --git a/ydb/core/tx/sequenceproxy/sequenceproxy_resolve.cpp b/ydb/core/tx/sequenceproxy/sequenceproxy_resolve.cpp index e69332cdf57..8dede913314 100644 --- a/ydb/core/tx/sequenceproxy/sequenceproxy_resolve.cpp +++ b/ydb/core/tx/sequenceproxy/sequenceproxy_resolve.cpp @@ -59,7 +59,7 @@ namespace NSequenceProxy { InitPath(entry, path); }, Path); - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; entry.SyncVersion = SyncVersion; Send(schemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(req.Release())); Become(&TThis::StateWork); diff --git a/ydb/core/tx/sequenceproxy/sequenceproxy_ut.cpp b/ydb/core/tx/sequenceproxy/sequenceproxy_ut.cpp index 5d10a01c93d..f08203fca0e 100644 --- a/ydb/core/tx/sequenceproxy/sequenceproxy_ut.cpp +++ b/ydb/core/tx/sequenceproxy/sequenceproxy_ut.cpp @@ -64,8 +64,8 @@ Y_UNIT_TEST_SUITE(SequenceProxy) { void StartSchemeCache(TTestActorRuntime& runtime, const TString& root = DOMAIN1_NAME) { for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) { - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); - cacheConfig->Roots.emplace_back(1, SCHEME_SHARD1_ID, root); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(); + cacheConfig->Roots.emplace_back(1, SCHEME_SHARD1_ID, root); cacheConfig->Counters = new NMonitoring::TDynamicCounters(); IActor* schemeCache = CreateSchemeBoardSchemeCache(cacheConfig.Get()); diff --git a/ydb/core/tx/tx_allocator_client/actor_client.cpp b/ydb/core/tx/tx_allocator_client/actor_client.cpp index 12409020cc9..20909a8401f 100644 --- a/ydb/core/tx/tx_allocator_client/actor_client.cpp +++ b/ydb/core/tx/tx_allocator_client/actor_client.cpp @@ -1,122 +1,122 @@ -#include "actor_client.h" -#include "client.h" - +#include "actor_client.h" +#include "client.h" + #include <ydb/core/base/tablet_pipe.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> - -#include <util/generic/deque.h> - -namespace NKikimr { - -class TTxAllocatorClientActor: public TActorBootstrapped<TTxAllocatorClientActor> { - struct TDelayedRequest { + +#include <util/generic/deque.h> + +namespace NKikimr { + +class TTxAllocatorClientActor: public TActorBootstrapped<TTxAllocatorClientActor> { + struct TDelayedRequest { TActorId Sender; - ui64 Cookie; + ui64 Cookie; ui64 Count; - }; - - static NTabletPipe::TClientConfig InitPipeClientConfig() { - NTabletPipe::TClientConfig config; + }; + + static NTabletPipe::TClientConfig InitPipeClientConfig() { + NTabletPipe::TClientConfig config; config.RetryPolicy = { .RetryLimitCount = 3, .MinRetryTime = TDuration::MilliSeconds(100), .MaxRetryTime = TDuration::Seconds(1), .BackoffMultiplier = 5 }; - return config; - } - - static const NTabletPipe::TClientConfig& GetPipeClientConfig() { - static const NTabletPipe::TClientConfig config = InitPipeClientConfig(); - return config; - } - - void Handle(TEvTxAllocatorClient::TEvAllocate::TPtr& ev, const TActorContext& ctx) { + return config; + } + + static const NTabletPipe::TClientConfig& GetPipeClientConfig() { + static const NTabletPipe::TClientConfig config = InitPipeClientConfig(); + return config; + } + + void Handle(TEvTxAllocatorClient::TEvAllocate::TPtr& ev, const TActorContext& ctx) { TVector<ui64> txIds = TxAllocatorClient.AllocateTxIds(ev->Get()->Count, ctx); - + if (txIds) { Send(ev->Sender, new TEvTxAllocatorClient::TEvAllocateResult(std::move(txIds)), 0, ev->Cookie); - } else { + } else { DelayedRequests.push_back({ev->Sender, ev->Cookie, ev->Get()->Count}); - } - } - - void Handle(TEvTxAllocator::TEvAllocateResult::TPtr& ev, const TActorContext& ctx) { - TxAllocatorClient.OnAllocateResult(ev, ctx); - - while (!DelayedRequests.empty()) { + } + } + + void Handle(TEvTxAllocator::TEvAllocateResult::TPtr& ev, const TActorContext& ctx) { + TxAllocatorClient.OnAllocateResult(ev, ctx); + + while (!DelayedRequests.empty()) { const TDelayedRequest& request = DelayedRequests.front(); - + TVector<ui64> txIds = TxAllocatorClient.AllocateTxIds(request.Count, ctx); if (!txIds) { - break; - } - + break; + } + Send(request.Sender, new TEvTxAllocatorClient::TEvAllocateResult(std::move(txIds)), 0, request.Cookie); - DelayedRequests.pop_front(); - } - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { - if (!PipeClientCache->OnConnect(ev)) { - TxAllocatorClient.SendRequest(ev->Get()->TabletId, ctx); - } - } - - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - PipeClientCache->OnDisconnect(ev); - TxAllocatorClient.SendRequest(ev->Get()->TabletId, ctx); - } - - void Die(const TActorContext& ctx) override { - PipeClientCache->Detach(ctx); - PipeClientCache.Destroy(); - - TActor::Die(ctx); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::TX_ALLOCATOR_CLIENT_ACTOR; - } - - explicit TTxAllocatorClientActor(const TVector<ui64>& txAllocators) - : PipeClientCache(NTabletPipe::CreateUnboundedClientCache(GetPipeClientConfig())) - , TxAllocatorClient(NKikimrServices::TX_ALLOCATOR_CLIENT, PipeClientCache.Get(), txAllocators) - { - } - - void Bootstrap(const TActorContext& ctx) { - TxAllocatorClient.Bootstrap(ctx); - Become(&TThis::StateWork); - } - - STFUNC(StateWork) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxAllocatorClient::TEvAllocate, Handle); - - HFunc(TEvTxAllocator::TEvAllocateResult, Handle); - - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - - CFunc(TEvents::TEvPoisonPill::EventType, Die); - } - } - -private: - THolder<NTabletPipe::IClientCache> PipeClientCache; - TTxAllocatorClient TxAllocatorClient; - - TDeque<TDelayedRequest> DelayedRequests; - -}; // TTxAllocatorClientActor - -IActor* CreateTxAllocatorClient(const TVector<ui64>& txAllocators) { - return new TTxAllocatorClientActor(txAllocators); -} - -} // NKikimr + DelayedRequests.pop_front(); + } + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { + if (!PipeClientCache->OnConnect(ev)) { + TxAllocatorClient.SendRequest(ev->Get()->TabletId, ctx); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { + PipeClientCache->OnDisconnect(ev); + TxAllocatorClient.SendRequest(ev->Get()->TabletId, ctx); + } + + void Die(const TActorContext& ctx) override { + PipeClientCache->Detach(ctx); + PipeClientCache.Destroy(); + + TActor::Die(ctx); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::TX_ALLOCATOR_CLIENT_ACTOR; + } + + explicit TTxAllocatorClientActor(const TVector<ui64>& txAllocators) + : PipeClientCache(NTabletPipe::CreateUnboundedClientCache(GetPipeClientConfig())) + , TxAllocatorClient(NKikimrServices::TX_ALLOCATOR_CLIENT, PipeClientCache.Get(), txAllocators) + { + } + + void Bootstrap(const TActorContext& ctx) { + TxAllocatorClient.Bootstrap(ctx); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxAllocatorClient::TEvAllocate, Handle); + + HFunc(TEvTxAllocator::TEvAllocateResult, Handle); + + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + + CFunc(TEvents::TEvPoisonPill::EventType, Die); + } + } + +private: + THolder<NTabletPipe::IClientCache> PipeClientCache; + TTxAllocatorClient TxAllocatorClient; + + TDeque<TDelayedRequest> DelayedRequests; + +}; // TTxAllocatorClientActor + +IActor* CreateTxAllocatorClient(const TVector<ui64>& txAllocators) { + return new TTxAllocatorClientActor(txAllocators); +} + +} // NKikimr diff --git a/ydb/core/tx/tx_allocator_client/actor_client.h b/ydb/core/tx/tx_allocator_client/actor_client.h index 66c3fdfdfb1..93d7caacce0 100644 --- a/ydb/core/tx/tx_allocator_client/actor_client.h +++ b/ydb/core/tx/tx_allocator_client/actor_client.h @@ -1,25 +1,25 @@ -#pragma once - -#include "defs.h" - -#include <util/generic/vector.h> - -namespace NKikimr { - -struct TEvTxAllocatorClient { - enum EEv { - EvAllocate = EventSpaceBegin(TKikimrEvents::ES_TX_ALLOCATOR_CLIENT), - EvAllocateResult, - - EvEnd, - }; - - static_assert( - EvEnd < EventSpaceEnd(TKikimrEvents::ES_TX_ALLOCATOR_CLIENT), - "expect End < EventSpaceEnd(ES_TX_ALLOCATOR_CLIENT)" - ); - - struct TEvAllocate: public TEventLocal<TEvAllocate, EvAllocate> { +#pragma once + +#include "defs.h" + +#include <util/generic/vector.h> + +namespace NKikimr { + +struct TEvTxAllocatorClient { + enum EEv { + EvAllocate = EventSpaceBegin(TKikimrEvents::ES_TX_ALLOCATOR_CLIENT), + EvAllocateResult, + + EvEnd, + }; + + static_assert( + EvEnd < EventSpaceEnd(TKikimrEvents::ES_TX_ALLOCATOR_CLIENT), + "expect End < EventSpaceEnd(ES_TX_ALLOCATOR_CLIENT)" + ); + + struct TEvAllocate: public TEventLocal<TEvAllocate, EvAllocate> { const ui32 Count = 1; TEvAllocate() = default; @@ -29,26 +29,26 @@ struct TEvTxAllocatorClient { { Y_VERIFY(count > 0); } - }; - - struct TEvAllocateResult: public TEventLocal<TEvAllocateResult, EvAllocateResult> { + }; + + struct TEvAllocateResult: public TEventLocal<TEvAllocateResult, EvAllocateResult> { const TVector<ui64> TxIds; - - TEvAllocateResult() = default; - - explicit TEvAllocateResult(ui64 txId) + + TEvAllocateResult() = default; + + explicit TEvAllocateResult(ui64 txId) : TxIds(1, txId) - { - } + { + } explicit TEvAllocateResult(TVector<ui64>&& txIds) : TxIds(std::move(txIds)) { } - }; - -}; // TTxAllocatorClientEvents - -IActor* CreateTxAllocatorClient(const TVector<ui64>& txAllocators); - -} // NKikimr + }; + +}; // TTxAllocatorClientEvents + +IActor* CreateTxAllocatorClient(const TVector<ui64>& txAllocators); + +} // NKikimr diff --git a/ydb/core/tx/tx_allocator_client/client.cpp b/ydb/core/tx/tx_allocator_client/client.cpp index 777b533fe19..d33144a166b 100644 --- a/ydb/core/tx/tx_allocator_client/client.cpp +++ b/ydb/core/tx/tx_allocator_client/client.cpp @@ -1,4 +1,4 @@ -#include "client.h" +#include "client.h" void NKikimr::TTxAllocatorClient::AddAllocationRange(const NKikimr::TTxAllocatorClient::TTabletId &from, const NKikimrTx::TEvTxAllocateResult &allocation) { ReservedRanges.emplace_back(from, allocation.GetRangeBegin(), allocation.GetRangeEnd()); diff --git a/ydb/core/tx/tx_allocator_client/client.h b/ydb/core/tx/tx_allocator_client/client.h index af370d2b9bf..d316e0eac2a 100644 --- a/ydb/core/tx/tx_allocator_client/client.h +++ b/ydb/core/tx/tx_allocator_client/client.h @@ -1,19 +1,19 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/protos/services.pb.h> #include <ydb/core/tablet/tablet_pipe_client_cache.h> #include <ydb/core/tx/tx_allocator/txallocator.h> - -#include <util/generic/ptr.h> -#include <util/generic/vector.h> - -namespace NKikimr { - -class TTxAllocatorClient { - using TTabletId = ui64; - + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> + +namespace NKikimr { + +class TTxAllocatorClient { + using TTabletId = ui64; + void RegisterRequest(const TTabletId& from, const TActorContext& ctx) { if (AllocationRequestsInFly.insert(from).second) { SendRequest(from, ctx); @@ -44,51 +44,51 @@ public: bool OnAllocateResult(TEvTxAllocator::TEvAllocateResult::TPtr& ev, const TActorContext& ctx); private: - struct TAllocationRange { - TTabletId Source = 0; - ui64 Begin = 0; - ui64 End = 0; - - TAllocationRange() = default; - - explicit TAllocationRange(const TTabletId& source, ui64 begin, ui64 end) - : Source(source) - , Begin(begin) - , End(end) - { - } - - TTabletId AllocatedFrom() const { - return Source; - } - - ui64 Capacity() const { - return End - Begin; - } - - bool Empty() const { - return Capacity() == 0; - } - - ui64 Allocate() { - Y_VERIFY(Capacity() >= 1); - return ++Begin; - } - }; - - NKikimrServices::EServiceKikimr Service; - NTabletPipe::IClientCache* PipeClientCache; - - TVector<TTabletId> TxAllocators; - THashSet<TTabletId> AllocationRequestsInFly; + struct TAllocationRange { + TTabletId Source = 0; + ui64 Begin = 0; + ui64 End = 0; + + TAllocationRange() = default; + + explicit TAllocationRange(const TTabletId& source, ui64 begin, ui64 end) + : Source(source) + , Begin(begin) + , End(end) + { + } + + TTabletId AllocatedFrom() const { + return Source; + } + + ui64 Capacity() const { + return End - Begin; + } + + bool Empty() const { + return Capacity() == 0; + } + + ui64 Allocate() { + Y_VERIFY(Capacity() >= 1); + return ++Begin; + } + }; + + NKikimrServices::EServiceKikimr Service; + NTabletPipe::IClientCache* PipeClientCache; + + TVector<TTabletId> TxAllocators; + THashSet<TTabletId> AllocationRequestsInFly; static constexpr ui64 BatchAllocationWarning = 500; - static constexpr ui64 RequestPerAllocator = 5000; - static constexpr ui64 PreRequestThreshhold = RequestPerAllocator / 3; - + static constexpr ui64 RequestPerAllocator = 5000; + static constexpr ui64 PreRequestThreshhold = RequestPerAllocator / 3; + const ui64 MaxCapacity = 0; ui64 Capacity = 0; TDeque<TAllocationRange> ReservedRanges; - -}; // TTxAllocatorClient - -} // NKikimr + +}; // TTxAllocatorClient + +} // NKikimr diff --git a/ydb/core/tx/tx_allocator_client/defs.h b/ydb/core/tx/tx_allocator_client/defs.h index 0fde5188877..bbae7d3179a 100644 --- a/ydb/core/tx/tx_allocator_client/defs.h +++ b/ydb/core/tx/tx_allocator_client/defs.h @@ -1,4 +1,4 @@ -#pragma once - +#pragma once + #include <ydb/core/base/defs.h> #include <ydb/core/base/events.h> diff --git a/ydb/core/tx/tx_allocator_client/ya.make b/ydb/core/tx/tx_allocator_client/ya.make index 63307d09eff..c4c4ab00eca 100644 --- a/ydb/core/tx/tx_allocator_client/ya.make +++ b/ydb/core/tx/tx_allocator_client/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( ydb/core/base ydb/core/protos ydb/core/tablet ydb/core/tx/tx_allocator -) - -SRCS( - actor_client.cpp - client.cpp -) - -END() +) + +SRCS( + actor_client.cpp + client.cpp +) + +END() diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 94fe9565fa9..af48b42d60f 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -1257,12 +1257,12 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte if (txbody.HasReadTableTransaction()) { ReadTableRequest = new TReadTableRequest(txbody.GetReadTableTransaction()); TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); - request->DatabaseName = record.GetDatabaseName(); + request->DatabaseName = record.GetDatabaseName(); NSchemeCache::TSchemeCacheNavigate::TEntry entry; entry.Path = SplitPath(ReadTableRequest->TablePath); entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; request->ResultSet.push_back(entry); ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); @@ -1365,7 +1365,7 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte NKikimrIssues::TStatusIds::TRANSIENT, false, ctx); } - resolveReq->Request->DatabaseName = record.GetDatabaseName(); + resolveReq->Request->DatabaseName = record.GetDatabaseName(); TxProxyMon->MakeRequestProxyAccepted->Inc(); LOG_DEBUG_S_SAMPLED_BY(ctx, NKikimrServices::TX_PROXY, TxId, "Actor# " << ctx.SelfID.ToString() << " txid# " << TxId @@ -1517,7 +1517,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, WallClockResolveStarted = Now(); TAutoPtr<NSchemeCache::TSchemeCacheRequest> request(new NSchemeCache::TSchemeCacheRequest); - request->DomainOwnerId = res.DomainInfo->ExtractSchemeShard(); + request->DomainOwnerId = res.DomainInfo->ExtractSchemeShard(); request->ResultSet.emplace_back(std::move(ReadTableRequest->KeyDesc)); ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvResolveKeySet(request)); } @@ -1606,25 +1606,25 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::AccessDenied, NKikimrIssues::TStatusIds::ACCESS_DENIED, true, ctx); return Die(ctx); } - - if (FlatMKQLRequest && entry.Kind == NSchemeCache::TSchemeCacheRequest::KindAsyncIndexTable) { - TMaybe<TString> error; - - if (entry.KeyDescription->RowOperation != TKeyDesc::ERowOperation::Read) { - error = TStringBuilder() << "Non-read operations can't be performed on async index table" - << ": " << entry.KeyDescription->TableId; + + if (FlatMKQLRequest && entry.Kind == NSchemeCache::TSchemeCacheRequest::KindAsyncIndexTable) { + TMaybe<TString> error; + + if (entry.KeyDescription->RowOperation != TKeyDesc::ERowOperation::Read) { + error = TStringBuilder() << "Non-read operations can't be performed on async index table" + << ": " << entry.KeyDescription->TableId; } else if (entry.KeyDescription->ReadTarget.GetMode() != TReadTarget::EMode::Follower) { - error = TStringBuilder() << "Read operation can be performed on async index table" - << ": " << entry.KeyDescription->TableId << " only with StaleRO isolation level"; - } - - if (error) { - LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, *error); - IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_TXPROXY_ERROR, *error)); - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecError, NKikimrIssues::TStatusIds::NOTSUPPORTED, true, ctx); - return Die(ctx); - } - } + error = TStringBuilder() << "Read operation can be performed on async index table" + << ": " << entry.KeyDescription->TableId << " only with StaleRO isolation level"; + } + + if (error) { + LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, *error); + IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_TXPROXY_ERROR, *error)); + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecError, NKikimrIssues::TStatusIds::NOTSUPPORTED, true, ctx); + return Die(ctx); + } + } } if (!CheckDomainLocality(*request)) { diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index a1882daf4d9..deb38ca2b7d 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -270,11 +270,11 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct } TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); - request->DatabaseName = record.GetDatabaseName(); + request->DatabaseName = record.GetDatabaseName(); NSchemeCache::TSchemeCacheNavigate::TEntry entry; entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; - entry.SyncVersion = true; - entry.ShowPrivatePath = record.GetDescribePath().GetOptions().GetShowPrivateTable(); + entry.SyncVersion = true; + entry.ShowPrivatePath = record.GetDescribePath().GetOptions().GetShowPrivateTable(); entry.Path = SplitPath(record.GetDescribePath().GetPath()); if (entry.Path.empty()) { ReportError(NKikimrScheme::StatusInvalidParameter, "Invalid path", ctx); @@ -284,7 +284,7 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct request->ResultSet.emplace_back(entry); - ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request), 0, SourceCookie); + ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request), 0, SourceCookie); SchemeRequest = ev->Release(); Become(&TThis::StateWaitResolve); @@ -363,7 +363,7 @@ void TDescribeReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr & return SendSystemViewResult(entry, path, ctx); } - const ui64 shardToRequest = entry.DomainInfo->ExtractSchemeShard(); + const ui64 shardToRequest = entry.DomainInfo->ExtractSchemeShard(); TAutoPtr<NSchemeShard::TEvSchemeShard::TEvDescribeScheme> req( new NSchemeShard::TEvSchemeShard::TEvDescribeScheme(describePath)); @@ -386,7 +386,7 @@ void TDescribeReq::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult: TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds()); - if (AppData()->FeatureFlags.GetEnableSystemViews()) { + if (AppData()->FeatureFlags.GetEnableSystemViews()) { const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription(); const auto& self = pathDescription.GetSelf(); diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h index 8c933dc2449..86d4f1ac79e 100644 --- a/ydb/core/tx/tx_proxy/proxy.h +++ b/ydb/core/tx/tx_proxy/proxy.h @@ -42,18 +42,18 @@ struct TEvTxUserProxy { EvInvalidateTable, EvInvalidateTableResult, - EvCancelBackupRequestDeprecated, - EvCancelBackupResultDeprecated, + EvCancelBackupRequestDeprecated, + EvCancelBackupResultDeprecated, EvProposeKqpTransaction, EvAllocateTxId, EvAllocateTxIdResult, - // deprecated - EvExportRequest = EvProposeTransaction + 2 * 512, - EvExportResponse, - + // deprecated + EvExportRequest = EvProposeTransaction + 2 * 512, + EvExportResponse, + EvUploadRowsResponse, EvGetProxyServicesRequest, @@ -177,7 +177,7 @@ struct TEvTxUserProxy { TEvProposeKqpTransaction(const TActorId& executerId) : ExecuterId(executerId) {} }; - + struct TEvAllocateTxId : public TEventLocal<TEvAllocateTxId, EvAllocateTxId> { // empty }; diff --git a/ydb/core/tx/tx_proxy/proxy_ext_tenant_ut.cpp b/ydb/core/tx/tx_proxy/proxy_ext_tenant_ut.cpp index b41bce3bbb7..ff7c9579d83 100644 --- a/ydb/core/tx/tx_proxy/proxy_ext_tenant_ut.cpp +++ b/ydb/core/tx/tx_proxy/proxy_ext_tenant_ut.cpp @@ -701,47 +701,47 @@ void GenericCases(TTestEnvWithPoolsSupport& env) { Y_UNIT_TEST_SUITE(TExtSubDomainTest) { Y_UNIT_TEST(DeclareAndLs) { - TTestEnv env(1, 0); + TTestEnv env(1, 0); DeclareAndLs(env); } Y_UNIT_TEST(DeclareAndDrop) { - TTestEnv env(1, 0); + TTestEnv env(1, 0); DeclareAndDrop(env); } Y_UNIT_TEST(DeclareAndDefineWithoutNodes) { - TTestEnvWithPoolsSupport env(1, 0, 2); + TTestEnvWithPoolsSupport env(1, 0, 2); DeclareAndDefineWithoutNodes(env); } Y_UNIT_TEST(DeclareAndDefineWithNodes) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); DeclareAndDefineWithNodes(env); } Y_UNIT_TEST(CreateTableInsideAndLs) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); CreateTableInsideAndLs(env); } Y_UNIT_TEST(DeclareAndAlterPools) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); DeclareAndAlterPools(env); } Y_UNIT_TEST(CreateTableInsideThenStopTenantAndForceDeleteSubDomain) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); CreateTableInsideThenStopTenantAndForceDeleteSubDomain(env); } Y_UNIT_TEST(CreateTableInsideAndAlterDomainAndTable) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); CreateTableInsideAndAlterDomainAndTable(env); } Y_UNIT_TEST(GenericCases) { - TTestEnvWithPoolsSupport env(1, 1, 2); + TTestEnvWithPoolsSupport env(1, 1, 2); GenericCases(env); } } diff --git a/ydb/core/tx/tx_proxy/proxy_impl.cpp b/ydb/core/tx/tx_proxy/proxy_impl.cpp index 9ee312393e0..20dc8cce176 100644 --- a/ydb/core/tx/tx_proxy/proxy_impl.cpp +++ b/ydb/core/tx/tx_proxy/proxy_impl.cpp @@ -47,7 +47,7 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { TTxProxyServices Services; THolder<NTabletPipe::IClientCache> PipeClientCache; - TTxAllocatorClient TxAllocatorClient; + TTxAllocatorClient TxAllocatorClient; static const TDuration TimeoutDelayedRequest; typedef TDelayedQueue<TEvTxUserProxy::TEvProposeTransaction> TDelayedProposal; @@ -198,7 +198,7 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { } void Handle(TEvTxAllocator::TEvAllocateResult::TPtr &ev, const TActorContext &ctx) { - if (!TxAllocatorClient.OnAllocateResult(ev, ctx)) { + if (!TxAllocatorClient.OnAllocateResult(ev, ctx)) { return; } @@ -214,7 +214,7 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { if(!PipeClientCache->OnConnect(ev)) { CheckTimeouts(ctx); - TxAllocatorClient.SendRequest(msg->TabletId, ctx); + TxAllocatorClient.SendRequest(msg->TabletId, ctx); return; } } @@ -228,7 +228,7 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { PipeClientCache->OnDisconnect(ev); CheckTimeouts(ctx); - TxAllocatorClient.SendRequest(msg->TabletId, ctx); + TxAllocatorClient.SendRequest(msg->TabletId, ctx); } void ProcessRequest(TEvTxUserProxy::TEvProposeTransaction::TPtr &ev, const TActorContext &ctx, ui64 txid) { @@ -422,9 +422,9 @@ class TTxProxy : public TActorBootstrapped<TTxProxy> { } public: - TTxProxy(const TVector<ui64> &txAllocators) - : PipeClientCache(NTabletPipe::CreateUnboundedClientCache(GetPipeClientConfig())) - , TxAllocatorClient(NKikimrServices::TX_PROXY, PipeClientCache.Get(), txAllocators) + TTxProxy(const TVector<ui64> &txAllocators) + : PipeClientCache(NTabletPipe::CreateUnboundedClientCache(GetPipeClientConfig())) + , TxAllocatorClient(NKikimrServices::TX_PROXY, PipeClientCache.Get(), txAllocators) { } @@ -437,15 +437,15 @@ public: Services.Proxy = SelfId(); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), CacheCounters); - Services.SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), CacheCounters); + Services.SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); ctx.ExecutorThread.ActorSystem->RegisterLocalService(MakeSchemeCacheID(), Services.SchemeCache); - + // PipePeNodeCaches are an external dependency Services.LeaderPipeCache = MakePipePeNodeCacheID(false); Services.FollowerPipeCache = MakePipePeNodeCacheID(true); - TxAllocatorClient.Bootstrap(ctx); + TxAllocatorClient.Bootstrap(ctx); Become(&TThis::StateWork); LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, diff --git a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h index 6676a578a68..846b0cf568c 100644 --- a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h +++ b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h @@ -95,7 +95,7 @@ protected: class TTestEnv: public TBaseTestEnv { public: - TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 0) + TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 0) { Settings = new Tests::TServerSettings(PortManager.GetPort(3534)); @@ -188,7 +188,7 @@ void SetRowInSimpletable(TBaseTestEnv& env, ui64 key, ui64 value, const TString class TTestEnvWithPoolsSupport: public TBaseTestEnv { public: - TTestEnvWithPoolsSupport(ui32 staticNodes = 1, ui32 dynamicNodes = 0, ui32 poolsCount = 2) + TTestEnvWithPoolsSupport(ui32 staticNodes = 1, ui32 dynamicNodes = 0, ui32 poolsCount = 2) { Settings = new Tests::TServerSettings(PortManager.GetPort(3534)); GetSettings().SetEnableMockOnSingleNode(false); diff --git a/ydb/core/tx/tx_proxy/read_table_impl.cpp b/ydb/core/tx/tx_proxy/read_table_impl.cpp index 0f5d28326dc..6c30975fd9b 100644 --- a/ydb/core/tx/tx_proxy/read_table_impl.cpp +++ b/ydb/core/tx/tx_proxy/read_table_impl.cpp @@ -471,7 +471,7 @@ private: auto& entry = request->ResultSet.emplace_back(); entry.Path = SplitPath(Settings.TablePath); entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; TXLOG_D("Sending TEvNagivateKeySet for table '" << Settings.TablePath << "'"); ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index a7ebafcd074..923fe381ae6 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -18,13 +18,13 @@ namespace NTxProxy { static constexpr TStringBuf DocApiRequestType = "_document_api_request"; static constexpr TStringBuf DocApiTableVersionAttribute = "__document_api_version"; - + template<typename TDerived> struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { using TBase = TActorBootstrapped<TDerived>; - + using TEvSchemeShardPropose = NSchemeShard::TEvSchemeShard::TEvModifySchemeTransaction; - + const TTxProxyServices Services; const ui64 TxId; THolder<TEvTxProxyReq::TEvSchemeRequest> SchemeRequest; @@ -78,7 +78,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { if (UserToken) { req->Record.SetUserToken(UserToken->SerializeAsString()); - } + } NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = {.RetryLimitCount = 3}; @@ -86,8 +86,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, "Actor# " << ctx.SelfID.ToString() << " txid# " << TxId << " SEND to# " << shardToRequest << " shardToRequest " << req->ToString()); NTabletPipe::SendData(ctx, PipeClient, req.Release()); - } - + } + static bool IsSplitMergeFromSchemeShard(const NKikimrSchemeOp::TModifyScheme& modifyScheme) { return modifyScheme.GetOperationType() == NKikimrSchemeOp::ESchemeOpSplitMergeTablePartitions && modifyScheme.GetSplitMergeTablePartitions().HasTableLocalId(); @@ -142,7 +142,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpDropColumnStore: case NKikimrSchemeOp::ESchemeOpDropColumnTable: case NKikimrSchemeOp::ESchemeOpDropSequence: - case NKikimrSchemeOp::ESchemeOpDropReplication: + case NKikimrSchemeOp::ESchemeOpDropReplication: return *modifyScheme.MutableDrop()->MutableName(); case NKikimrSchemeOp::ESchemeOpAlterTable: @@ -196,29 +196,29 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpCreateConsistentCopyTables: Y_FAIL("no implementation for ESchemeOpCreateConsistentCopyTables"); - + case NKikimrSchemeOp::ESchemeOp_DEPRECATED_35: Y_FAIL("no implementation for ESchemeOp_DEPRECATED_35"); - + case NKikimrSchemeOp::ESchemeOpUpgradeSubDomain: case NKikimrSchemeOp::ESchemeOpUpgradeSubDomainDecision: return *modifyScheme.MutableUpgradeSubDomain()->MutableName(); - + case NKikimrSchemeOp::ESchemeOpCreateIndexBuild: Y_FAIL("no implementation for ESchemeOpCreateIndexBuild"); - + case NKikimrSchemeOp::ESchemeOpInitiateBuildIndexMainTable: Y_FAIL("no implementation for ESchemeOpInitiateBuildIndexMainTable"); - + case NKikimrSchemeOp::ESchemeOpCreateLockForIndexBuild: Y_FAIL("no implementation for ESchemeOpCreateLockForIndexBuild"); - + case NKikimrSchemeOp::ESchemeOpApplyIndexBuild: Y_FAIL("no implementation for ESchemeOpApplyIndexBuild"); - + case NKikimrSchemeOp::ESchemeOpFinalizeBuildIndexMainTable: Y_FAIL("no implementation for ESchemeOpFinalizeBuildIndexMainTable"); - + case NKikimrSchemeOp::ESchemeOpAlterTableIndex: return *modifyScheme.MutableAlterTableIndex()->MutableName(); @@ -304,9 +304,9 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpAlterSequence: return *modifyScheme.MutableSequence()->MutableName(); - case NKikimrSchemeOp::ESchemeOpCreateReplication: - case NKikimrSchemeOp::ESchemeOpAlterReplication: - return *modifyScheme.MutableReplication()->MutableName(); + case NKikimrSchemeOp::ESchemeOpCreateReplication: + case NKikimrSchemeOp::ESchemeOpAlterReplication: + return *modifyScheme.MutableReplication()->MutableName(); } } @@ -338,27 +338,27 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { static THolder<NSchemeCache::TSchemeCacheNavigate> ResolveRequestForAdjustPathNames(NKikimrSchemeOp::TModifyScheme& scheme) { auto parts = Merge(SplitPath(scheme.GetWorkingDir()), SplitPath(GetPathNameForScheme(scheme))); - if (parts.size() < 2) { + if (parts.size() < 2) { return {}; - } - + } + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - TVector<TString> path; - - for (auto it = parts.begin(); it != parts.end() - 1; ++it) { - path.emplace_back(*it); - + TVector<TString> path; + + for (auto it = parts.begin(); it != parts.end() - 1; ++it) { + path.emplace_back(*it); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = path; + entry.Path = path; entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpPath; - entry.SyncVersion = true; - entry.RedirectRequired = false; - request->ResultSet.emplace_back(entry); - } - + entry.SyncVersion = true; + entry.RedirectRequired = false; + request->ResultSet.emplace_back(entry); + } + return std::move(request); - } - + } + void ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus status, const NKikimrScheme::TEvModifySchemeTransactionResult* shardResult, const NYql::TIssue* issue, @@ -371,10 +371,10 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { if (shardResult) { if (shardResult->HasTxId()) result->Record.SetTxId(shardResult->GetTxId()); - + if (shardResult->HasSchemeshardId()) result->Record.SetSchemeShardTabletId(shardResult->GetSchemeshardId()); - + result->Record.SetSchemeShardStatus(shardResult->GetStatus()); if (shardResult->HasReason() && !shardResult->GetReason().empty()) @@ -545,7 +545,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpAlterColumnStore: case NKikimrSchemeOp::ESchemeOpAlterColumnTable: case NKikimrSchemeOp::ESchemeOpAlterSequence: - case NKikimrSchemeOp::ESchemeOpAlterReplication: + case NKikimrSchemeOp::ESchemeOpAlterReplication: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); @@ -562,8 +562,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpDropSolomonVolume: case NKikimrSchemeOp::ESchemeOpDropColumnStore: case NKikimrSchemeOp::ESchemeOpDropColumnTable: - case NKikimrSchemeOp::ESchemeOpDropSequence: - case NKikimrSchemeOp::ESchemeOpDropReplication: { + case NKikimrSchemeOp::ESchemeOpDropSequence: + case NKikimrSchemeOp::ESchemeOpDropReplication: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); toResolve.RequiredAccess = NACLib::EAccessRights::RemoveSchema; @@ -619,7 +619,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpCreateKesus: case NKikimrSchemeOp::ESchemeOpCreateSolomonVolume: case NKikimrSchemeOp::ESchemeOpCreateSequence: - case NKikimrSchemeOp::ESchemeOpCreateReplication: + case NKikimrSchemeOp::ESchemeOpCreateReplication: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = workingDir; @@ -733,7 +733,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { entry.Path = toReq.Path; entry.RedirectRequired = toReq.RequiredRedirect; entry.SyncVersion = true; - entry.ShowPrivatePath = true; + entry.ShowPrivatePath = true; request->ResultSet.emplace_back(std::move(entry)); } @@ -763,8 +763,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { } } - void InterpretResolveError(const NSchemeCache::TSchemeCacheNavigate* navigate, const TActorContext &ctx) { - for (const auto& entry: navigate->ResultSet) { + void InterpretResolveError(const NSchemeCache::TSchemeCacheNavigate* navigate, const TActorContext &ctx) { + for (const auto& entry: navigate->ResultSet) { switch (entry.Status) { case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: continue; @@ -811,13 +811,13 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { return; } - - LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, "Unexpected response from scheme cache" - << ": " << navigate->ToString(*AppData()->TypeRegistry)); - Y_VERIFY_DEBUG(false, "Unreachable"); - - TxProxyMon->ResolveKeySetFail->Inc(); - ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, ctx); + + LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, "Unexpected response from scheme cache" + << ": " << navigate->ToString(*AppData()->TypeRegistry)); + Y_VERIFY_DEBUG(false, "Unreachable"); + + TxProxyMon->ResolveKeySetFail->Inc(); + ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, ctx); } static bool IsDB(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry) { @@ -998,7 +998,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { Y_VERIFY(!navigate->ResultSet.empty()); if (navigate->ErrorCount > 0) { - InterpretResolveError(navigate, ctx); + InterpretResolveError(navigate, ctx); return Die(ctx); } @@ -1149,44 +1149,44 @@ void TFlatSchemeReq::ProcessRequest(const TActorContext &ctx) { return; } -void TFlatSchemeReq::HandleWorkingDir(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, +void TFlatSchemeReq::HandleWorkingDir(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, "Actor# " << ctx.SelfID.ToString() - << " txid# " << TxId - << " HANDLE EvNavigateKeySetResult TFlatSchemeReq marker# P6"); + << " txid# " << TxId + << " HANDLE EvNavigateKeySetResult TFlatSchemeReq marker# P6"); Y_VERIFY(NeedAdjustPathNames(GetModifyScheme())); - + const auto& resultSet = ev->Get()->Request->ResultSet; - - const TVector<TString>* workingDir = nullptr; - for (auto it = resultSet.rbegin(); it != resultSet.rend(); ++it) { - if (it->Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { - workingDir = &it->Path; - break; - } - } - + + const TVector<TString>* workingDir = nullptr; + for (auto it = resultSet.rbegin(); it != resultSet.rend(); ++it) { + if (it->Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { + workingDir = &it->Path; + break; + } + } + auto parts = Merge(SplitPath(GetModifyScheme().GetWorkingDir()), SplitPath(GetPathNameForScheme(GetModifyScheme()))); - - if (!workingDir || workingDir->size() >= parts.size()) { + + if (!workingDir || workingDir->size() >= parts.size()) { const TString errText = TStringBuilder() << "Cannot resolve working dir" << " workingDir# " << (workingDir ? JoinPath(*workingDir) : "null") << " path# " << JoinPath(parts); LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, errText); - - TxProxyMon->ResolveKeySetWrongRequest->Inc(); + + TxProxyMon->ResolveKeySetWrongRequest->Inc(); const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, errText); ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, nullptr, &issue, ctx); - return Die(ctx); - } - + return Die(ctx); + } + GetModifyScheme().SetWorkingDir(CombinePath(workingDir->begin(), workingDir->end())); GetPathNameForScheme(GetModifyScheme()) = CombinePath(parts.begin() + workingDir->size(), parts.end(), false); - ProcessRequest(ctx); -} - + ProcessRequest(ctx); +} + ////////////////////////////////////////////////////////////// /// \brief The TFlatSchemeTransactionalReq struct /// struct for Trasaction with several ModifySchema @@ -1219,7 +1219,7 @@ struct TSchemeTransactionalReq : public TBaseSchemeReq<TSchemeTransactionalReq> HFunc(NSchemeShard::TEvSchemeShard::TEvModifySchemeTransactionResult, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - } + } } }; @@ -1249,9 +1249,9 @@ void TSchemeTransactionalReq::Bootstrap(const TActorContext &ctx) { ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::NotImplemented, ctx); TxProxyMon->ResolveKeySetWrongRequest->Inc(); return Die(ctx); - } - } - + } + } + auto resolveRequest = ResolveRequestForACL(); if (!resolveRequest) { ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, ctx); diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 69833f4d099..0aaee7e7f59 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -280,19 +280,19 @@ private: auto& entry = ResolveNamesResult->ResultSet.front(); - for (const auto& index : entry.Indexes) { - switch (index.GetType()) { - case NKikimrSchemeOp::EIndexTypeGlobalAsync: - if (AppData(ctx)->FeatureFlags.GetEnableBulkUpsertToAsyncIndexedTables()) { - continue; - } else { - errorMessage = "Bulk upsert is not supported for tables with indexes"; - return false; - } - default: - errorMessage = "Only async-indexed tables are supported by BulkUpsert"; - return false; - } + for (const auto& index : entry.Indexes) { + switch (index.GetType()) { + case NKikimrSchemeOp::EIndexTypeGlobalAsync: + if (AppData(ctx)->FeatureFlags.GetEnableBulkUpsertToAsyncIndexedTables()) { + continue; + } else { + errorMessage = "Bulk upsert is not supported for tables with indexes"; + return false; + } + default: + errorMessage = "Only async-indexed tables are supported by BulkUpsert"; + return false; + } } TVector<ui32> keyColumnIds; @@ -429,8 +429,8 @@ private: return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, "Invalid table path specified", ctx); } entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - entry.SyncVersion = true; - entry.ShowPrivatePath = AllowWriteToPrivateTable; + entry.SyncVersion = true; + entry.ShowPrivatePath = AllowWriteToPrivateTable; request->ResultSet.emplace_back(entry); ctx.Send(SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); diff --git a/ydb/core/tx/ya.make b/ydb/core/tx/ya.make index 1a0d5c5346e..5bfe4b3b861 100644 --- a/ydb/core/tx/ya.make +++ b/ydb/core/tx/ya.make @@ -34,7 +34,7 @@ RECURSE( datashard long_tx_service mediator - replication + replication scheme_board scheme_cache schemeshard diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h index 806c73aa5be..c5716f46a76 100644 --- a/ydb/core/viewer/browse.h +++ b/ydb/core/viewer/browse.h @@ -85,11 +85,11 @@ public: case NKikimrSchemeOp::EPathType::EPathTypeSolomonVolume: return NKikimrViewer::EObjectType::SolomonVolume; case NKikimrSchemeOp::EPathType::EPathTypeCdcStream: - return NKikimrViewer::EObjectType::CdcStream; + return NKikimrViewer::EObjectType::CdcStream; case NKikimrSchemeOp::EPathType::EPathTypeSequence: return NKikimrViewer::EObjectType::Sequence; - case NKikimrSchemeOp::EPathType::EPathTypeReplication: - return NKikimrViewer::EObjectType::Replication; + case NKikimrSchemeOp::EPathType::EPathTypeReplication: + return NKikimrViewer::EObjectType::Replication; case NKikimrSchemeOp::EPathType::EPathTypeExtSubDomain: case NKikimrSchemeOp::EPathType::EPathTypeTableIndex: case NKikimrSchemeOp::EPathType::EPathTypeInvalid: diff --git a/ydb/core/viewer/content/v2/tablet_cell.js b/ydb/core/viewer/content/v2/tablet_cell.js index 9e6972f0cf2..4129ba9dd94 100644 --- a/ydb/core/viewer/content/v2/tablet_cell.js +++ b/ydb/core/viewer/content/v2/tablet_cell.js @@ -24,7 +24,7 @@ TabletCell.prototype.tabletTypes = { 'BlockStorePartition': 'BP', 'BlockStoreVolume': 'BV', 'SequenceShard': 'S', - 'ReplicationController': 'RC', + 'ReplicationController': 'RC', }; TabletCell.prototype.getTabletType = function() { diff --git a/ydb/core/viewer/content/viewer.js b/ydb/core/viewer/content/viewer.js index a90c1e8b89a..1108f42278c 100644 --- a/ydb/core/viewer/content/viewer.js +++ b/ydb/core/viewer/content/viewer.js @@ -1504,8 +1504,8 @@ function tabletTypeToSymbol(type) { return "CS"; case "SequenceShard": return "S"; - case "ReplicationController": - return "RC"; + case "ReplicationController": + return "RC"; } return "XX"; } @@ -2500,20 +2500,20 @@ function onTreeDataComplete(result, obj, cb) { } obj.describe = result; } - if (result.PathDescription.Table.CdcStreams) { - var ilen = result.PathDescription.Table.CdcStreams.length; - for (var i = 0; i < ilen; i++) { - var child = {}; - child.id = concatPaths(result.Path, result.PathDescription.Table.CdcStreams[i].Name); - child.parent = result.Path; - child.text = result.PathDescription.Table.CdcStreams[i].Name; - child.children = true; - child.icon = "glyphicon glyphicon-book schema-good"; - child.data = function(obj1, cb1) { onTreeData(obj1, cb1); }; - children.push(child); - } - obj.describe = result; - } + if (result.PathDescription.Table.CdcStreams) { + var ilen = result.PathDescription.Table.CdcStreams.length; + for (var i = 0; i < ilen; i++) { + var child = {}; + child.id = concatPaths(result.Path, result.PathDescription.Table.CdcStreams[i].Name); + child.parent = result.Path; + child.text = result.PathDescription.Table.CdcStreams[i].Name; + child.children = true; + child.icon = "glyphicon glyphicon-book schema-good"; + child.data = function(obj1, cb1) { onTreeData(obj1, cb1); }; + children.push(child); + } + obj.describe = result; + } } cb.call(this, children); } else { @@ -2594,8 +2594,8 @@ function schemaPathTypeToString(pathType) { return "Tenant"; case 15: return "Sequence"; - case 16: - return "Replication"; + case 16: + return "Replication"; } } @@ -2732,7 +2732,7 @@ function onTreeNodeComplete(result, obj) { schemaHtml += "</table>"; panelSchema.innerHTML = schemaHtml; - var indexes = table.TableIndexes; + var indexes = table.TableIndexes; if (indexes !== undefined) { var ilen = indexes.length; var schemaHtml = "<br><table class='schema-table'><tr><th>IndexName</th><th>Columns</th></tr>"; @@ -2750,20 +2750,20 @@ function onTreeNodeComplete(result, obj) { schemaHtml += "</table>"; panelSchema.innerHTML += schemaHtml; } - - var cdcStreams = table.CdcStreams; - if (cdcStreams !== undefined) { - var ilen = cdcStreams.length; - var schemaHtml = "<br><table class='schema-table'><tr><th>StreamName</th><th>Mode</th></tr>"; - for (var i = 0; i < ilen; i++) { - schemaHtml += "<tr>"; - schemaHtml += "<td>" + cdcStreams[i].Name + "</td>"; - schemaHtml += "<td>" + cdcStreams[i].Mode + "</td>"; - schemaHtml += "</tr>" - } - schemaHtml += "</table>"; - panelSchema.innerHTML += schemaHtml; - } + + var cdcStreams = table.CdcStreams; + if (cdcStreams !== undefined) { + var ilen = cdcStreams.length; + var schemaHtml = "<br><table class='schema-table'><tr><th>StreamName</th><th>Mode</th></tr>"; + for (var i = 0; i < ilen; i++) { + schemaHtml += "<tr>"; + schemaHtml += "<td>" + cdcStreams[i].Name + "</td>"; + schemaHtml += "<td>" + cdcStreams[i].Mode + "</td>"; + schemaHtml += "</tr>" + } + schemaHtml += "</table>"; + panelSchema.innerHTML += schemaHtml; + } } var tabLen; @@ -3095,11 +3095,11 @@ function onTreeNodeComplete(result, obj) { row.insertCell(-1).innerHTML = "DomainKey"; row.insertCell(-1).innerHTML = result.PathDescription.DomainDescription.DomainKey.SchemeShard + " : " + result.PathDescription.DomainDescription.DomainKey.PathId; } - if (result.PathDescription.DomainDescription.ResourcesDomainKey !== undefined) { - row = tab.insertRow(); - row.insertCell(-1).innerHTML = "ResourcesDomainKey"; - row.insertCell(-1).innerHTML = result.PathDescription.DomainDescription.ResourcesDomainKey.SchemeShard + " : " + result.PathDescription.DomainDescription.ResourcesDomainKey.PathId; - } + if (result.PathDescription.DomainDescription.ResourcesDomainKey !== undefined) { + row = tab.insertRow(); + row.insertCell(-1).innerHTML = "ResourcesDomainKey"; + row.insertCell(-1).innerHTML = result.PathDescription.DomainDescription.ResourcesDomainKey.SchemeShard + " : " + result.PathDescription.DomainDescription.ResourcesDomainKey.PathId; + } if (result.PathDescription.DomainDescription.PathsInside !== undefined) { row = tab.insertRow(); row.insertCell(-1).innerHTML = "Pathes inside"; diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index cc41acb37b9..9f6c822a89f 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -31,9 +31,9 @@ enum EObjectType { SolomonVolume = 17; SubDomain = 18; FileStore = 19; - CdcStream = 20; + CdcStream = 20; Sequence = 21; - Replication = 22; + Replication = 22; } message TBrowseInfo { diff --git a/ydb/core/wrappers/s3_out.cpp b/ydb/core/wrappers/s3_out.cpp index bffb4997d77..0bc2745a6d0 100644 --- a/ydb/core/wrappers/s3_out.cpp +++ b/ydb/core/wrappers/s3_out.cpp @@ -1,173 +1,173 @@ -#if defined(KIKIMR_DISABLE_S3_WRAPPER) -#error "s3 wrapper is disabled" -#endif - -#include "s3_out.h" - -#include <util/generic/vector.h> -#include <util/string/join.h> - -namespace NKikimr { -namespace NWrappers { - -using namespace Aws::S3::Model; - -#define DEFINE_PRINTER(field) \ - template <typename T> \ - void field(IOutputStream& out, const T& obj) { \ - out << #field ": " << obj.Get##field(); \ - } - -DEFINE_PRINTER(Bucket) -DEFINE_PRINTER(Key) -DEFINE_PRINTER(UploadId) -DEFINE_PRINTER(MultipartUpload) -DEFINE_PRINTER(PartNumber) -DEFINE_PRINTER(ETag) -DEFINE_PRINTER(ContentLength) -DEFINE_PRINTER(Range) - -#undef DEFINE_PRINTER - -template <typename T> -using TPrinter = std::function<void(IOutputStream&, const T&)>; - -template <typename T> -void OutRequest(IOutputStream& out, const T& request, const TVector<TPrinter<T>>& printers) { - out << request.GetServiceRequestName() << " {"; - for (const auto& printer : printers) { - out << " "; - printer(out, request); - } - out << " }"; -} - -template <typename T> -void OutResult(IOutputStream& out, const T& result, TStringBuf header, const TVector<TPrinter<T>>& printers) { - out << header << " {"; - for (const auto& printer : printers) { - out << " "; - printer(out, result); - } - out << " }"; -} - -template <typename T> -void OutOutcome(IOutputStream& out, const Aws::Utils::Outcome<T, Aws::S3::S3Error>& outcome) { - if (outcome.IsSuccess()) { - out << outcome.GetResult(); - } else { - out << outcome.GetError().GetMessage().c_str(); - } -} - -void Out(IOutputStream& out, const GetObjectRequest& request) { - using T = GetObjectRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>, &Range<T>}); -} - -void Out(IOutputStream& out, const GetObjectResult& result) { - OutResult(out, result, "GetObjectResult", {}); -} - -void Out(IOutputStream& out, const GetObjectOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const HeadObjectRequest& request) { - using T = HeadObjectRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>}); -} - -void Out(IOutputStream& out, const HeadObjectResult& result) { - using T = HeadObjectResult; - OutResult(out, result, "HeadObjectResult", {&ETag<T>, &ContentLength<T>}); -} - -void Out(IOutputStream& out, const HeadObjectOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const PutObjectRequest& request) { - using T = PutObjectRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>}); -} - -void Out(IOutputStream& out, const PutObjectResult& result) { - using T = PutObjectResult; - OutResult(out, result, "PutObjectResult", {&ETag<T>}); -} - -void Out(IOutputStream& out, const PutObjectOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const CreateMultipartUploadRequest& request) { - using T = CreateMultipartUploadRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>}); -} - -void Out(IOutputStream& out, const CreateMultipartUploadResult& result) { - using T = CreateMultipartUploadResult; - OutResult(out, result, "CreateMultipartUploadResult", {&Bucket<T>, &Key<T>, &UploadId<T>}); -} - -void Out(IOutputStream& out, const CreateMultipartUploadOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const CompleteMultipartUploadRequest& request) { - using T = CompleteMultipartUploadRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>, &MultipartUpload<T>}); -} - -void Out(IOutputStream& out, const CompleteMultipartUploadResult& result) { - using T = CompleteMultipartUploadResult; - OutResult(out, result, "CompleteMultipartUploadResult", {&Bucket<T>, &Key<T>, &ETag<T>}); -} - -void Out(IOutputStream& out, const CompleteMultipartUploadOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const AbortMultipartUploadRequest& request) { - using T = AbortMultipartUploadRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>}); -} - -void Out(IOutputStream& out, const AbortMultipartUploadResult& result) { - OutResult(out, result, "AbortMultipartUploadResult", {}); -} - -void Out(IOutputStream& out, const AbortMultipartUploadOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const UploadPartRequest& request) { - using T = UploadPartRequest; - OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>, &PartNumber<T>}); -} - -void Out(IOutputStream& out, const UploadPartResult& result) { - using T = UploadPartResult; - OutResult(out, result, "UploadPartResult", {&ETag<T>}); -} - -void Out(IOutputStream& out, const UploadPartOutcome& outcome) { - OutOutcome(out, outcome); -} - -void Out(IOutputStream& out, const CompletedMultipartUpload& upload) { - out << "{ Parts: [" << JoinSeq(",", upload.GetParts()) << "] }"; -} - -void Out(IOutputStream& out, const CompletedPart& part) { - out << part.GetETag(); -} - -void Out(IOutputStream& out, const TStringOutcome& outcome) { - OutOutcome(out, outcome); -} - -} // NWrappers -} // NKikimr +#if defined(KIKIMR_DISABLE_S3_WRAPPER) +#error "s3 wrapper is disabled" +#endif + +#include "s3_out.h" + +#include <util/generic/vector.h> +#include <util/string/join.h> + +namespace NKikimr { +namespace NWrappers { + +using namespace Aws::S3::Model; + +#define DEFINE_PRINTER(field) \ + template <typename T> \ + void field(IOutputStream& out, const T& obj) { \ + out << #field ": " << obj.Get##field(); \ + } + +DEFINE_PRINTER(Bucket) +DEFINE_PRINTER(Key) +DEFINE_PRINTER(UploadId) +DEFINE_PRINTER(MultipartUpload) +DEFINE_PRINTER(PartNumber) +DEFINE_PRINTER(ETag) +DEFINE_PRINTER(ContentLength) +DEFINE_PRINTER(Range) + +#undef DEFINE_PRINTER + +template <typename T> +using TPrinter = std::function<void(IOutputStream&, const T&)>; + +template <typename T> +void OutRequest(IOutputStream& out, const T& request, const TVector<TPrinter<T>>& printers) { + out << request.GetServiceRequestName() << " {"; + for (const auto& printer : printers) { + out << " "; + printer(out, request); + } + out << " }"; +} + +template <typename T> +void OutResult(IOutputStream& out, const T& result, TStringBuf header, const TVector<TPrinter<T>>& printers) { + out << header << " {"; + for (const auto& printer : printers) { + out << " "; + printer(out, result); + } + out << " }"; +} + +template <typename T> +void OutOutcome(IOutputStream& out, const Aws::Utils::Outcome<T, Aws::S3::S3Error>& outcome) { + if (outcome.IsSuccess()) { + out << outcome.GetResult(); + } else { + out << outcome.GetError().GetMessage().c_str(); + } +} + +void Out(IOutputStream& out, const GetObjectRequest& request) { + using T = GetObjectRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>, &Range<T>}); +} + +void Out(IOutputStream& out, const GetObjectResult& result) { + OutResult(out, result, "GetObjectResult", {}); +} + +void Out(IOutputStream& out, const GetObjectOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const HeadObjectRequest& request) { + using T = HeadObjectRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>}); +} + +void Out(IOutputStream& out, const HeadObjectResult& result) { + using T = HeadObjectResult; + OutResult(out, result, "HeadObjectResult", {&ETag<T>, &ContentLength<T>}); +} + +void Out(IOutputStream& out, const HeadObjectOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const PutObjectRequest& request) { + using T = PutObjectRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>}); +} + +void Out(IOutputStream& out, const PutObjectResult& result) { + using T = PutObjectResult; + OutResult(out, result, "PutObjectResult", {&ETag<T>}); +} + +void Out(IOutputStream& out, const PutObjectOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const CreateMultipartUploadRequest& request) { + using T = CreateMultipartUploadRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>}); +} + +void Out(IOutputStream& out, const CreateMultipartUploadResult& result) { + using T = CreateMultipartUploadResult; + OutResult(out, result, "CreateMultipartUploadResult", {&Bucket<T>, &Key<T>, &UploadId<T>}); +} + +void Out(IOutputStream& out, const CreateMultipartUploadOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const CompleteMultipartUploadRequest& request) { + using T = CompleteMultipartUploadRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>, &MultipartUpload<T>}); +} + +void Out(IOutputStream& out, const CompleteMultipartUploadResult& result) { + using T = CompleteMultipartUploadResult; + OutResult(out, result, "CompleteMultipartUploadResult", {&Bucket<T>, &Key<T>, &ETag<T>}); +} + +void Out(IOutputStream& out, const CompleteMultipartUploadOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const AbortMultipartUploadRequest& request) { + using T = AbortMultipartUploadRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>}); +} + +void Out(IOutputStream& out, const AbortMultipartUploadResult& result) { + OutResult(out, result, "AbortMultipartUploadResult", {}); +} + +void Out(IOutputStream& out, const AbortMultipartUploadOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const UploadPartRequest& request) { + using T = UploadPartRequest; + OutRequest(out, request, {&Bucket<T>, &Key<T>, &UploadId<T>, &PartNumber<T>}); +} + +void Out(IOutputStream& out, const UploadPartResult& result) { + using T = UploadPartResult; + OutResult(out, result, "UploadPartResult", {&ETag<T>}); +} + +void Out(IOutputStream& out, const UploadPartOutcome& outcome) { + OutOutcome(out, outcome); +} + +void Out(IOutputStream& out, const CompletedMultipartUpload& upload) { + out << "{ Parts: [" << JoinSeq(",", upload.GetParts()) << "] }"; +} + +void Out(IOutputStream& out, const CompletedPart& part) { + out << part.GetETag(); +} + +void Out(IOutputStream& out, const TStringOutcome& outcome) { + OutOutcome(out, outcome); +} + +} // NWrappers +} // NKikimr diff --git a/ydb/core/wrappers/s3_out.h b/ydb/core/wrappers/s3_out.h index f966973fdbb..ac6143592f8 100644 --- a/ydb/core/wrappers/s3_out.h +++ b/ydb/core/wrappers/s3_out.h @@ -1,148 +1,148 @@ -#pragma once - -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/AbortMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CreateMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CompleteMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/GetObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/HeadObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/PutObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/UploadPartRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/S3Client.h> - -#include <util/stream/output.h> - -namespace NKikimr { -namespace NWrappers { - -void Out(IOutputStream& out, const Aws::S3::Model::GetObjectRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::GetObjectResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::GetObjectOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::PutObjectRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::PutObjectResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::PutObjectOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::UploadPartRequest& request); -void Out(IOutputStream& out, const Aws::S3::Model::UploadPartResult& result); -void Out(IOutputStream& out, const Aws::S3::Model::UploadPartOutcome& outcome); - -void Out(IOutputStream& out, const Aws::S3::Model::CompletedMultipartUpload& upload); -void Out(IOutputStream& out, const Aws::S3::Model::CompletedPart& part); - -using TStringOutcome = Aws::Utils::Outcome<Aws::String, Aws::S3::S3Error>; -void Out(IOutputStream& out, const TStringOutcome& outcome); - -} // NWrappers -} // NKikimr - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartRequest, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartResult, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompletedMultipartUpload, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompletedPart, out, value) { - NKikimr::NWrappers::Out(out, value); -} - -Y_DECLARE_OUT_SPEC(inline, NKikimr::NWrappers::TStringOutcome, out, value) { - NKikimr::NWrappers::Out(out, value); -} +#pragma once + +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/AbortMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CreateMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CompleteMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/GetObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/HeadObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/PutObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/UploadPartRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/S3Client.h> + +#include <util/stream/output.h> + +namespace NKikimr { +namespace NWrappers { + +void Out(IOutputStream& out, const Aws::S3::Model::GetObjectRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::GetObjectResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::GetObjectOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::HeadObjectOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::PutObjectRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::PutObjectResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::PutObjectOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::CreateMultipartUploadOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::CompleteMultipartUploadOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::AbortMultipartUploadOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::UploadPartRequest& request); +void Out(IOutputStream& out, const Aws::S3::Model::UploadPartResult& result); +void Out(IOutputStream& out, const Aws::S3::Model::UploadPartOutcome& outcome); + +void Out(IOutputStream& out, const Aws::S3::Model::CompletedMultipartUpload& upload); +void Out(IOutputStream& out, const Aws::S3::Model::CompletedPart& part); + +using TStringOutcome = Aws::Utils::Outcome<Aws::String, Aws::S3::S3Error>; +void Out(IOutputStream& out, const TStringOutcome& outcome); + +} // NWrappers +} // NKikimr + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::GetObjectOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::HeadObjectOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::PutObjectOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CreateMultipartUploadOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompleteMultipartUploadOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::AbortMultipartUploadOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartRequest, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartResult, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::UploadPartOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompletedMultipartUpload, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, Aws::S3::Model::CompletedPart, out, value) { + NKikimr::NWrappers::Out(out, value); +} + +Y_DECLARE_OUT_SPEC(inline, NKikimr::NWrappers::TStringOutcome, out, value) { + NKikimr::NWrappers::Out(out, value); +} diff --git a/ydb/core/wrappers/s3_wrapper.cpp b/ydb/core/wrappers/s3_wrapper.cpp index 9e9ec8bddbb..b0fe378e218 100644 --- a/ydb/core/wrappers/s3_wrapper.cpp +++ b/ydb/core/wrappers/s3_wrapper.cpp @@ -1,352 +1,352 @@ -#if defined(KIKIMR_DISABLE_S3_WRAPPER) -#error "s3 wrapper is disabled" -#endif - -#include "s3_wrapper.h" - -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/internal/AWSHttpResourceClient.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/stream/PreallocatedStreamBuf.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/stream/ResponseStream.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/Aws.h> -#include <contrib/libs/curl/include/curl/curl.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - -#include <util/generic/singleton.h> -#include <util/string/cast.h> -#include <util/system/mutex.h> - -namespace NKikimr { -namespace NWrappers { - -using namespace Aws; -using namespace Aws::Auth; -using namespace Aws::Client; -using namespace Aws::S3; -using namespace Aws::S3::Model; -using namespace Aws::Utils::Stream; - -namespace { - - struct TCurlInitializer { - TCurlInitializer() { - curl_global_init(CURL_GLOBAL_ALL); - } - - ~TCurlInitializer() { - curl_global_cleanup(); - } - }; - - struct TApiInitializer { - TApiInitializer() { - Options.httpOptions.initAndCleanupCurl = false; - InitAPI(Options); - - Internal::CleanupEC2MetadataClient(); // speeds up config construction - } - - ~TApiInitializer() { - ShutdownAPI(Options); - } - - private: - SDKOptions Options; - }; - - class TApiOwner { - public: - void Ref() { - auto guard = Guard(Mutex); - if (!RefCount++) { - if (!CurlInitializer) { - CurlInitializer.Reset(new TCurlInitializer); - } - ApiInitializer.Reset(new TApiInitializer); - } - } - - void UnRef() { - auto guard = Guard(Mutex); - if (!--RefCount) { - ApiInitializer.Destroy(); - } - } - - private: - ui64 RefCount = 0; - TMutex Mutex; - THolder<TCurlInitializer> CurlInitializer; - THolder<TApiInitializer> ApiInitializer; - }; - +#if defined(KIKIMR_DISABLE_S3_WRAPPER) +#error "s3 wrapper is disabled" +#endif + +#include "s3_wrapper.h" + +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/internal/AWSHttpResourceClient.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/stream/PreallocatedStreamBuf.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/utils/stream/ResponseStream.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/Aws.h> +#include <contrib/libs/curl/include/curl/curl.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/generic/singleton.h> +#include <util/string/cast.h> +#include <util/system/mutex.h> + +namespace NKikimr { +namespace NWrappers { + +using namespace Aws; +using namespace Aws::Auth; +using namespace Aws::Client; +using namespace Aws::S3; +using namespace Aws::S3::Model; +using namespace Aws::Utils::Stream; + +namespace { + + struct TCurlInitializer { + TCurlInitializer() { + curl_global_init(CURL_GLOBAL_ALL); + } + + ~TCurlInitializer() { + curl_global_cleanup(); + } + }; + + struct TApiInitializer { + TApiInitializer() { + Options.httpOptions.initAndCleanupCurl = false; + InitAPI(Options); + + Internal::CleanupEC2MetadataClient(); // speeds up config construction + } + + ~TApiInitializer() { + ShutdownAPI(Options); + } + + private: + SDKOptions Options; + }; + + class TApiOwner { + public: + void Ref() { + auto guard = Guard(Mutex); + if (!RefCount++) { + if (!CurlInitializer) { + CurlInitializer.Reset(new TCurlInitializer); + } + ApiInitializer.Reset(new TApiInitializer); + } + } + + void UnRef() { + auto guard = Guard(Mutex); + if (!--RefCount) { + ApiInitializer.Destroy(); + } + } + + private: + ui64 RefCount = 0; + TMutex Mutex; + THolder<TCurlInitializer> CurlInitializer; + THolder<TApiInitializer> ApiInitializer; + }; + } // anonymous - + TS3User::TS3User() { Singleton<TApiOwner>()->Ref(); } - + TS3User::~TS3User() { Singleton<TApiOwner>()->UnRef(); } - + class TS3Wrapper: public TActor<TS3Wrapper>, private TS3User { - template <typename TEvRequest, typename TEvResponse> - class TContextBase: public AsyncCallerContext { - public: - explicit TContextBase(const TActorSystem* sys, const TActorId& sender) - : AsyncCallerContext() - , ActorSystem(sys) - , Sender(sender) - , Replied(false) - { - } - - const TActorSystem* GetActorSystem() const { - return ActorSystem; - } - - virtual const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) { - return ev->Get()->Request; - } - - void Reply(const typename TEvResponse::TOutcome& outcome) const { - Y_VERIFY(!std::exchange(Replied, true), "Double-reply"); - Send(MakeResponse(outcome).Release()); - } - - protected: - void Send(const TActorId& recipient, IEventBase* ev) const { - ActorSystem->Send(recipient, ev); - } - - void Send(IEventBase* ev) const { - Send(Sender, ev); - } - - virtual THolder<IEventBase> MakeResponse(const typename TEvResponse::TOutcome& outcome) const { + template <typename TEvRequest, typename TEvResponse> + class TContextBase: public AsyncCallerContext { + public: + explicit TContextBase(const TActorSystem* sys, const TActorId& sender) + : AsyncCallerContext() + , ActorSystem(sys) + , Sender(sender) + , Replied(false) + { + } + + const TActorSystem* GetActorSystem() const { + return ActorSystem; + } + + virtual const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) { + return ev->Get()->Request; + } + + void Reply(const typename TEvResponse::TOutcome& outcome) const { + Y_VERIFY(!std::exchange(Replied, true), "Double-reply"); + Send(MakeResponse(outcome).Release()); + } + + protected: + void Send(const TActorId& recipient, IEventBase* ev) const { + ActorSystem->Send(recipient, ev); + } + + void Send(IEventBase* ev) const { + Send(Sender, ev); + } + + virtual THolder<IEventBase> MakeResponse(const typename TEvResponse::TOutcome& outcome) const { return MakeHolder<TEvResponse>(outcome); - } - - private: - const TActorSystem* ActorSystem; - const TActorId Sender; - - mutable bool Replied; - - }; // TContextBase - - template <typename TEvRequest, typename TEvResponse> - class TOutputStreamContext: public TContextBase<TEvRequest, TEvResponse> { - class TOutputStreamBuf: public PreallocatedStreamBuf { - TOutputStreamBuf(char* data, size_t size) - : PreallocatedStreamBuf(reinterpret_cast<unsigned char*>(data), size) - { - } - - public: - explicit TOutputStreamBuf(TString& buffer) - : TOutputStreamBuf(buffer.Detach(), buffer.Size()) - { - } - }; - - static bool TryParseRange(const TString& str, std::pair<ui64, ui64>& range) { - TStringBuf buf(str); - if (!buf.SkipPrefix("bytes=")) { - return false; - } - - ui64 start; - if (!TryFromString(buf.NextTok('-'), start)) { - return false; - } - - ui64 end; - if (!TryFromString(buf, end)) { - return false; - } - - range = std::make_pair(start, end); - return true; - } - - public: - using TContextBase<TEvRequest, TEvResponse>::TContextBase; - - const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) override { - auto& request = ev->Get()->Request; - - std::pair<ui64, ui64> range; - Y_VERIFY(request.RangeHasBeenSet() && TryParseRange(request.GetRange().c_str(), range)); - - Buffer.resize(range.second - range.first + 1); - request.SetResponseStreamFactory([this]() { - return Aws::New<DefaultUnderlyingStream>("StreamContext", - MakeUnique<TOutputStreamBuf>("StreamContext", Buffer)); - }); - - return request; - } - - protected: - THolder<IEventBase> MakeResponse(const typename TEvResponse::TOutcome& outcome) const override { - if (outcome.IsSuccess()) { + } + + private: + const TActorSystem* ActorSystem; + const TActorId Sender; + + mutable bool Replied; + + }; // TContextBase + + template <typename TEvRequest, typename TEvResponse> + class TOutputStreamContext: public TContextBase<TEvRequest, TEvResponse> { + class TOutputStreamBuf: public PreallocatedStreamBuf { + TOutputStreamBuf(char* data, size_t size) + : PreallocatedStreamBuf(reinterpret_cast<unsigned char*>(data), size) + { + } + + public: + explicit TOutputStreamBuf(TString& buffer) + : TOutputStreamBuf(buffer.Detach(), buffer.Size()) + { + } + }; + + static bool TryParseRange(const TString& str, std::pair<ui64, ui64>& range) { + TStringBuf buf(str); + if (!buf.SkipPrefix("bytes=")) { + return false; + } + + ui64 start; + if (!TryFromString(buf.NextTok('-'), start)) { + return false; + } + + ui64 end; + if (!TryFromString(buf, end)) { + return false; + } + + range = std::make_pair(start, end); + return true; + } + + public: + using TContextBase<TEvRequest, TEvResponse>::TContextBase; + + const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) override { + auto& request = ev->Get()->Request; + + std::pair<ui64, ui64> range; + Y_VERIFY(request.RangeHasBeenSet() && TryParseRange(request.GetRange().c_str(), range)); + + Buffer.resize(range.second - range.first + 1); + request.SetResponseStreamFactory([this]() { + return Aws::New<DefaultUnderlyingStream>("StreamContext", + MakeUnique<TOutputStreamBuf>("StreamContext", Buffer)); + }); + + return request; + } + + protected: + THolder<IEventBase> MakeResponse(const typename TEvResponse::TOutcome& outcome) const override { + if (outcome.IsSuccess()) { return MakeHolder<TEvResponse>(outcome, std::move(Buffer)); - } else { + } else { return MakeHolder<TEvResponse>(outcome); - } - } - - private: - mutable TString Buffer; - - }; // TOutputStreamContext - - template <typename TEvRequest, typename TEvResponse> - class TInputStreamContext: public TContextBase<TEvRequest, TEvResponse> { - class TInputStreamBuf: public PreallocatedStreamBuf { - TInputStreamBuf(char* data, size_t size) - : PreallocatedStreamBuf(reinterpret_cast<unsigned char*>(data), size) - { - } - - TInputStreamBuf(const char* data, size_t size) - : TInputStreamBuf(const_cast<char*>(data), size) - { - } - - public: - explicit TInputStreamBuf(const TStringBuf buf) - : TInputStreamBuf(buf.data(), buf.size()) - { - } - }; - - public: - using TContextBase<TEvRequest, TEvResponse>::TContextBase; - - const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) override { - auto& request = ev->Get()->Request; - - Buffer = std::move(ev->Get()->Body); - request.SetBody(MakeShared<DefaultUnderlyingStream>("StreamContext", - MakeUnique<TInputStreamBuf>("StreamContext", Buffer))); - - return request; - } - - private: - TString Buffer; - - }; // TInputStreamContext - - template <typename TRequest, typename TOutcome> - using THandler = std::function<void(const S3Client*, const TRequest&, const TOutcome&, const std::shared_ptr<const AsyncCallerContext>&)>; - - template <typename TRequest, typename TOutcome> - using TFunc = std::function<void(const S3Client*, const TRequest&, THandler<TRequest, TOutcome>, const std::shared_ptr<const AsyncCallerContext>&)>; - - template <typename TEvRequest, typename TEvResponse, template <typename...> typename TContext = TContextBase> - void Call(typename TEvRequest::TPtr& ev, TFunc<typename TEvRequest::TRequest, typename TEvResponse::TOutcome> func) { - using TCtx = TContext<TEvRequest, TEvResponse>; - - auto ctx = std::make_shared<TCtx>(TlsActivationContext->ActorSystem(), ev->Sender); - auto callback = []( - const S3Client*, - const typename TEvRequest::TRequest&, - const typename TEvResponse::TOutcome& outcome, - const std::shared_ptr<const AsyncCallerContext>& context) { - const auto* ctx = static_cast<const TCtx*>(context.get()); - - LOG_NOTICE_S(*ctx->GetActorSystem(), NKikimrServices::S3_WRAPPER, "Response" - << ": uuid# " << ctx->GetUUID() - << ", response# " << outcome); - ctx->Reply(outcome); - }; - - LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::S3_WRAPPER, "Request" - << ": uuid# " << ctx->GetUUID() - << ", request# " << ev->Get()->Request); - func(Client.Get(), ctx->PrepareRequest(ev), callback, ctx); - } - - void Handle(TEvS3Wrapper::TEvGetObjectRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvGetObjectRequest, TEvS3Wrapper::TEvGetObjectResponse, TOutputStreamContext>( - ev, &S3Client::GetObjectAsync); - } - - void Handle(TEvS3Wrapper::TEvHeadObjectRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvHeadObjectRequest, TEvS3Wrapper::TEvHeadObjectResponse>( - ev, &S3Client::HeadObjectAsync); - } - - void Handle(TEvS3Wrapper::TEvPutObjectRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvPutObjectRequest, TEvS3Wrapper::TEvPutObjectResponse, TInputStreamContext>( - ev, &S3Client::PutObjectAsync); - } - - void Handle(TEvS3Wrapper::TEvCreateMultipartUploadRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvCreateMultipartUploadRequest, TEvS3Wrapper::TEvCreateMultipartUploadResponse>( - ev, &S3Client::CreateMultipartUploadAsync); - } - - void Handle(TEvS3Wrapper::TEvUploadPartRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvUploadPartRequest, TEvS3Wrapper::TEvUploadPartResponse, TInputStreamContext>( - ev, &S3Client::UploadPartAsync); - } - - void Handle(TEvS3Wrapper::TEvCompleteMultipartUploadRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvCompleteMultipartUploadRequest, TEvS3Wrapper::TEvCompleteMultipartUploadResponse>( - ev, &S3Client::CompleteMultipartUploadAsync); - } - - void Handle(TEvS3Wrapper::TEvAbortMultipartUploadRequest::TPtr& ev) { - Call<TEvS3Wrapper::TEvAbortMultipartUploadRequest, TEvS3Wrapper::TEvAbortMultipartUploadResponse>( - ev, &S3Client::AbortMultipartUploadAsync); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::S3_WRAPPER_ACTOR; - } - - explicit TS3Wrapper(const AWSCredentials& credentials, const ClientConfiguration& config) - : TActor(&TThis::StateWork) - , Client(new S3Client(credentials, config)) - { - } - - virtual ~TS3Wrapper() { - if (Client) { - Client->DisableRequestProcessing(); - } - } - - STATEFN(StateWork) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvS3Wrapper::TEvGetObjectRequest, Handle); - hFunc(TEvS3Wrapper::TEvHeadObjectRequest, Handle); - hFunc(TEvS3Wrapper::TEvPutObjectRequest, Handle); - hFunc(TEvS3Wrapper::TEvCreateMultipartUploadRequest, Handle); - hFunc(TEvS3Wrapper::TEvUploadPartRequest, Handle); - hFunc(TEvS3Wrapper::TEvCompleteMultipartUploadRequest, Handle); - hFunc(TEvS3Wrapper::TEvAbortMultipartUploadRequest, Handle); - - cFunc(TEvents::TEvPoison::EventType, PassAway); - } - } - -private: - THolder<S3Client> Client; - -}; // TS3Wrapper - -IActor* CreateS3Wrapper(const AWSCredentials& credentials, const ClientConfiguration& config) { - return new TS3Wrapper(credentials, config); -} - -} // NWrappers -} // NKikimr + } + } + + private: + mutable TString Buffer; + + }; // TOutputStreamContext + + template <typename TEvRequest, typename TEvResponse> + class TInputStreamContext: public TContextBase<TEvRequest, TEvResponse> { + class TInputStreamBuf: public PreallocatedStreamBuf { + TInputStreamBuf(char* data, size_t size) + : PreallocatedStreamBuf(reinterpret_cast<unsigned char*>(data), size) + { + } + + TInputStreamBuf(const char* data, size_t size) + : TInputStreamBuf(const_cast<char*>(data), size) + { + } + + public: + explicit TInputStreamBuf(const TStringBuf buf) + : TInputStreamBuf(buf.data(), buf.size()) + { + } + }; + + public: + using TContextBase<TEvRequest, TEvResponse>::TContextBase; + + const typename TEvRequest::TRequest& PrepareRequest(typename TEvRequest::TPtr& ev) override { + auto& request = ev->Get()->Request; + + Buffer = std::move(ev->Get()->Body); + request.SetBody(MakeShared<DefaultUnderlyingStream>("StreamContext", + MakeUnique<TInputStreamBuf>("StreamContext", Buffer))); + + return request; + } + + private: + TString Buffer; + + }; // TInputStreamContext + + template <typename TRequest, typename TOutcome> + using THandler = std::function<void(const S3Client*, const TRequest&, const TOutcome&, const std::shared_ptr<const AsyncCallerContext>&)>; + + template <typename TRequest, typename TOutcome> + using TFunc = std::function<void(const S3Client*, const TRequest&, THandler<TRequest, TOutcome>, const std::shared_ptr<const AsyncCallerContext>&)>; + + template <typename TEvRequest, typename TEvResponse, template <typename...> typename TContext = TContextBase> + void Call(typename TEvRequest::TPtr& ev, TFunc<typename TEvRequest::TRequest, typename TEvResponse::TOutcome> func) { + using TCtx = TContext<TEvRequest, TEvResponse>; + + auto ctx = std::make_shared<TCtx>(TlsActivationContext->ActorSystem(), ev->Sender); + auto callback = []( + const S3Client*, + const typename TEvRequest::TRequest&, + const typename TEvResponse::TOutcome& outcome, + const std::shared_ptr<const AsyncCallerContext>& context) { + const auto* ctx = static_cast<const TCtx*>(context.get()); + + LOG_NOTICE_S(*ctx->GetActorSystem(), NKikimrServices::S3_WRAPPER, "Response" + << ": uuid# " << ctx->GetUUID() + << ", response# " << outcome); + ctx->Reply(outcome); + }; + + LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::S3_WRAPPER, "Request" + << ": uuid# " << ctx->GetUUID() + << ", request# " << ev->Get()->Request); + func(Client.Get(), ctx->PrepareRequest(ev), callback, ctx); + } + + void Handle(TEvS3Wrapper::TEvGetObjectRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvGetObjectRequest, TEvS3Wrapper::TEvGetObjectResponse, TOutputStreamContext>( + ev, &S3Client::GetObjectAsync); + } + + void Handle(TEvS3Wrapper::TEvHeadObjectRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvHeadObjectRequest, TEvS3Wrapper::TEvHeadObjectResponse>( + ev, &S3Client::HeadObjectAsync); + } + + void Handle(TEvS3Wrapper::TEvPutObjectRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvPutObjectRequest, TEvS3Wrapper::TEvPutObjectResponse, TInputStreamContext>( + ev, &S3Client::PutObjectAsync); + } + + void Handle(TEvS3Wrapper::TEvCreateMultipartUploadRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvCreateMultipartUploadRequest, TEvS3Wrapper::TEvCreateMultipartUploadResponse>( + ev, &S3Client::CreateMultipartUploadAsync); + } + + void Handle(TEvS3Wrapper::TEvUploadPartRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvUploadPartRequest, TEvS3Wrapper::TEvUploadPartResponse, TInputStreamContext>( + ev, &S3Client::UploadPartAsync); + } + + void Handle(TEvS3Wrapper::TEvCompleteMultipartUploadRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvCompleteMultipartUploadRequest, TEvS3Wrapper::TEvCompleteMultipartUploadResponse>( + ev, &S3Client::CompleteMultipartUploadAsync); + } + + void Handle(TEvS3Wrapper::TEvAbortMultipartUploadRequest::TPtr& ev) { + Call<TEvS3Wrapper::TEvAbortMultipartUploadRequest, TEvS3Wrapper::TEvAbortMultipartUploadResponse>( + ev, &S3Client::AbortMultipartUploadAsync); + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::S3_WRAPPER_ACTOR; + } + + explicit TS3Wrapper(const AWSCredentials& credentials, const ClientConfiguration& config) + : TActor(&TThis::StateWork) + , Client(new S3Client(credentials, config)) + { + } + + virtual ~TS3Wrapper() { + if (Client) { + Client->DisableRequestProcessing(); + } + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvS3Wrapper::TEvGetObjectRequest, Handle); + hFunc(TEvS3Wrapper::TEvHeadObjectRequest, Handle); + hFunc(TEvS3Wrapper::TEvPutObjectRequest, Handle); + hFunc(TEvS3Wrapper::TEvCreateMultipartUploadRequest, Handle); + hFunc(TEvS3Wrapper::TEvUploadPartRequest, Handle); + hFunc(TEvS3Wrapper::TEvCompleteMultipartUploadRequest, Handle); + hFunc(TEvS3Wrapper::TEvAbortMultipartUploadRequest, Handle); + + cFunc(TEvents::TEvPoison::EventType, PassAway); + } + } + +private: + THolder<S3Client> Client; + +}; // TS3Wrapper + +IActor* CreateS3Wrapper(const AWSCredentials& credentials, const ClientConfiguration& config) { + return new TS3Wrapper(credentials, config); +} + +} // NWrappers +} // NKikimr diff --git a/ydb/core/wrappers/s3_wrapper.h b/ydb/core/wrappers/s3_wrapper.h index 7da760818f0..1197606202b 100644 --- a/ydb/core/wrappers/s3_wrapper.h +++ b/ydb/core/wrappers/s3_wrapper.h @@ -1,175 +1,175 @@ -#pragma once - -#include "s3_out.h" - -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/auth/AWSCredentials.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/AbortMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CreateMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CompleteMultipartUploadRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/GetObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/HeadObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/PutObjectRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/UploadPartRequest.h> -#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/S3Client.h> - +#pragma once + +#include "s3_out.h" + +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/auth/AWSCredentials.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/AbortMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CreateMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/CompleteMultipartUploadRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/GetObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/HeadObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/PutObjectRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/UploadPartRequest.h> +#include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/S3Client.h> + #include <ydb/core/base/events.h> - -namespace NKikimr { -namespace NWrappers { - + +namespace NKikimr { +namespace NWrappers { + struct TS3User { TS3User(); ~TS3User(); }; -struct TEvS3Wrapper { - template <typename TDerived, ui32 EventType, typename T> - struct TGenericRequest: public TEventLocal<TDerived, EventType> { - using TRequest = T; - TRequest Request; - - explicit TGenericRequest(const TRequest& request) - : Request(request) - { - } - - using TBase = TGenericRequest<TDerived, EventType, TRequest>; - }; - - template <typename TDerived, ui32 EventType, typename T> - struct TRequestWithBody: public TGenericRequest<TDerived, EventType, T> { - using TGeneric = TGenericRequest<TDerived, EventType, T>; - - TString Body; - - explicit TRequestWithBody(const typename TGeneric::TRequest& request, TString&& body) - : TGeneric(request) - , Body(std::move(body)) - { - } - - using TBase = TRequestWithBody<TDerived, EventType, T>; - }; - - template <typename TDerived, ui32 EventType, typename T, typename U = T> - struct TGenericResponse: public TEventLocal<TDerived, EventType> { +struct TEvS3Wrapper { + template <typename TDerived, ui32 EventType, typename T> + struct TGenericRequest: public TEventLocal<TDerived, EventType> { + using TRequest = T; + TRequest Request; + + explicit TGenericRequest(const TRequest& request) + : Request(request) + { + } + + using TBase = TGenericRequest<TDerived, EventType, TRequest>; + }; + + template <typename TDerived, ui32 EventType, typename T> + struct TRequestWithBody: public TGenericRequest<TDerived, EventType, T> { + using TGeneric = TGenericRequest<TDerived, EventType, T>; + + TString Body; + + explicit TRequestWithBody(const typename TGeneric::TRequest& request, TString&& body) + : TGeneric(request) + , Body(std::move(body)) + { + } + + using TBase = TRequestWithBody<TDerived, EventType, T>; + }; + + template <typename TDerived, ui32 EventType, typename T, typename U = T> + struct TGenericResponse: public TEventLocal<TDerived, EventType> { using TOutcome = Aws::Utils::Outcome<T, Aws::S3::S3Error>; using TResult = Aws::Utils::Outcome<U, Aws::S3::S3Error>; - - TResult Result; - - explicit TGenericResponse(const TOutcome& outcome) - : Result(TDerived::ResultFromOutcome(outcome)) - { - } - - static TResult ResultFromOutcome(const TOutcome& outcome) { - return outcome; - } - - using TBase = TGenericResponse<TDerived, EventType, T, U>; - }; - - template <typename TDerived, ui32 EventType, typename T, typename U> - struct TResponseWithBody: public TGenericResponse<TDerived, EventType, T, U> { - using TGeneric = TGenericResponse<TDerived, EventType, T, U>; - - TString Body; - - explicit TResponseWithBody(const typename TGeneric::TOutcome& outcome) - : TGeneric(outcome) - { - } - - explicit TResponseWithBody(const typename TGeneric::TOutcome& outcome, TString&& body) - : TGeneric(outcome) - , Body(std::move(body)) - { - } - - using TBase = TResponseWithBody<TDerived, EventType, T, U>; - }; - - #define EV_REQUEST_RESPONSE(name) \ - Ev##name##Request, \ - Ev##name##Response - - enum EEv { - EvBegin = EventSpaceBegin(TKikimrEvents::ES_S3_WRAPPER), - - EV_REQUEST_RESPONSE(GetObject), - EV_REQUEST_RESPONSE(HeadObject), - EV_REQUEST_RESPONSE(PutObject), - EV_REQUEST_RESPONSE(CreateMultipartUpload), - EV_REQUEST_RESPONSE(UploadPart), - EV_REQUEST_RESPONSE(CompleteMultipartUpload), - EV_REQUEST_RESPONSE(AbortMultipartUpload), - - EvEnd, - }; - - #undef EV_REQUEST_RESPONSE - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_S3_WRAPPER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_S3_WRAPPER)"); - - #define DEFINE_REQUEST(name, base) \ - struct TEv##name##Request: public base<TEv##name##Request, Ev##name##Request, Aws::S3::Model::name##Request> { \ - using TBase::TBase; \ - } - - #define DEFINE_GENERIC_REQUEST(name) \ - DEFINE_REQUEST(name, TGenericRequest) - - #define DECLARE_GENERIC_RESPONSE(name) \ - struct TEv##name##Response: public TGenericResponse<TEv##name##Response, Ev##name##Response, Aws::S3::Model::name##Result> - - #define DECLARE_RESPONSE_WITH_BODY(name, result_t) \ - struct TEv##name##Response: public TResponseWithBody<TEv##name##Response, Ev##name##Response, Aws::S3::Model::name##Result, result_t> - - #define DEFINE_GENERIC_RESPONSE(name) \ - DECLARE_GENERIC_RESPONSE(name) { \ - using TBase::TBase; \ - } - - #define DEFINE_GENERIC_REQUEST_RESPONSE(name) \ - DEFINE_GENERIC_REQUEST(name); \ - DEFINE_GENERIC_RESPONSE(name) - - DEFINE_GENERIC_REQUEST(GetObject); - DECLARE_RESPONSE_WITH_BODY(GetObject, Aws::String) { - static TResult ResultFromOutcome(const TOutcome& outcome) { - if (outcome.IsSuccess()) { - return outcome.GetResult().GetETag(); - } else { - return outcome.GetError(); - } - } - - using TBase::TBase; - }; - - DEFINE_REQUEST(PutObject, TRequestWithBody); - DEFINE_GENERIC_RESPONSE(PutObject); - - DEFINE_REQUEST(UploadPart, TRequestWithBody); - DEFINE_GENERIC_RESPONSE(UploadPart); - - DEFINE_GENERIC_REQUEST_RESPONSE(HeadObject); - DEFINE_GENERIC_REQUEST_RESPONSE(CreateMultipartUpload); - DEFINE_GENERIC_REQUEST_RESPONSE(CompleteMultipartUpload); - DEFINE_GENERIC_REQUEST_RESPONSE(AbortMultipartUpload); - - #undef DEFINE_REQUEST - #undef DEFINE_GENERIC_REQUEST - #undef DECLARE_GENERIC_RESPONSE - #undef DECLARE_RESPONSE_WITH_BODY - #undef DEFINE_GENERIC_RESPONSE - #undef DEFINE_GENERIC_REQUEST_RESPONSE - -}; // TEvS3Wrapper - -IActor* CreateS3Wrapper(const Aws::Auth::AWSCredentials& credentials, const Aws::Client::ClientConfiguration& config); - -} // NWrappers -} // NKikimr + + TResult Result; + + explicit TGenericResponse(const TOutcome& outcome) + : Result(TDerived::ResultFromOutcome(outcome)) + { + } + + static TResult ResultFromOutcome(const TOutcome& outcome) { + return outcome; + } + + using TBase = TGenericResponse<TDerived, EventType, T, U>; + }; + + template <typename TDerived, ui32 EventType, typename T, typename U> + struct TResponseWithBody: public TGenericResponse<TDerived, EventType, T, U> { + using TGeneric = TGenericResponse<TDerived, EventType, T, U>; + + TString Body; + + explicit TResponseWithBody(const typename TGeneric::TOutcome& outcome) + : TGeneric(outcome) + { + } + + explicit TResponseWithBody(const typename TGeneric::TOutcome& outcome, TString&& body) + : TGeneric(outcome) + , Body(std::move(body)) + { + } + + using TBase = TResponseWithBody<TDerived, EventType, T, U>; + }; + + #define EV_REQUEST_RESPONSE(name) \ + Ev##name##Request, \ + Ev##name##Response + + enum EEv { + EvBegin = EventSpaceBegin(TKikimrEvents::ES_S3_WRAPPER), + + EV_REQUEST_RESPONSE(GetObject), + EV_REQUEST_RESPONSE(HeadObject), + EV_REQUEST_RESPONSE(PutObject), + EV_REQUEST_RESPONSE(CreateMultipartUpload), + EV_REQUEST_RESPONSE(UploadPart), + EV_REQUEST_RESPONSE(CompleteMultipartUpload), + EV_REQUEST_RESPONSE(AbortMultipartUpload), + + EvEnd, + }; + + #undef EV_REQUEST_RESPONSE + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_S3_WRAPPER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_S3_WRAPPER)"); + + #define DEFINE_REQUEST(name, base) \ + struct TEv##name##Request: public base<TEv##name##Request, Ev##name##Request, Aws::S3::Model::name##Request> { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_REQUEST(name) \ + DEFINE_REQUEST(name, TGenericRequest) + + #define DECLARE_GENERIC_RESPONSE(name) \ + struct TEv##name##Response: public TGenericResponse<TEv##name##Response, Ev##name##Response, Aws::S3::Model::name##Result> + + #define DECLARE_RESPONSE_WITH_BODY(name, result_t) \ + struct TEv##name##Response: public TResponseWithBody<TEv##name##Response, Ev##name##Response, Aws::S3::Model::name##Result, result_t> + + #define DEFINE_GENERIC_RESPONSE(name) \ + DECLARE_GENERIC_RESPONSE(name) { \ + using TBase::TBase; \ + } + + #define DEFINE_GENERIC_REQUEST_RESPONSE(name) \ + DEFINE_GENERIC_REQUEST(name); \ + DEFINE_GENERIC_RESPONSE(name) + + DEFINE_GENERIC_REQUEST(GetObject); + DECLARE_RESPONSE_WITH_BODY(GetObject, Aws::String) { + static TResult ResultFromOutcome(const TOutcome& outcome) { + if (outcome.IsSuccess()) { + return outcome.GetResult().GetETag(); + } else { + return outcome.GetError(); + } + } + + using TBase::TBase; + }; + + DEFINE_REQUEST(PutObject, TRequestWithBody); + DEFINE_GENERIC_RESPONSE(PutObject); + + DEFINE_REQUEST(UploadPart, TRequestWithBody); + DEFINE_GENERIC_RESPONSE(UploadPart); + + DEFINE_GENERIC_REQUEST_RESPONSE(HeadObject); + DEFINE_GENERIC_REQUEST_RESPONSE(CreateMultipartUpload); + DEFINE_GENERIC_REQUEST_RESPONSE(CompleteMultipartUpload); + DEFINE_GENERIC_REQUEST_RESPONSE(AbortMultipartUpload); + + #undef DEFINE_REQUEST + #undef DEFINE_GENERIC_REQUEST + #undef DECLARE_GENERIC_RESPONSE + #undef DECLARE_RESPONSE_WITH_BODY + #undef DEFINE_GENERIC_RESPONSE + #undef DEFINE_GENERIC_REQUEST_RESPONSE + +}; // TEvS3Wrapper + +IActor* CreateS3Wrapper(const Aws::Auth::AWSCredentials& credentials, const Aws::Client::ClientConfiguration& config); + +} // NWrappers +} // NKikimr diff --git a/ydb/core/wrappers/s3_wrapper_ut.cpp b/ydb/core/wrappers/s3_wrapper_ut.cpp index 487e16b5d7c..04cb00e56de 100644 --- a/ydb/core/wrappers/s3_wrapper_ut.cpp +++ b/ydb/core/wrappers/s3_wrapper_ut.cpp @@ -3,284 +3,284 @@ #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/wrappers/ut_helpers/s3_mock.h> #include <ydb/core/wrappers/s3_wrapper.h> - -#include <library/cpp/actors/core/log.h> -#include <library/cpp/digest/md5/md5.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <util/string/printf.h> - -using namespace NActors; -using namespace NKikimr; -using namespace Aws::S3::Model; - -class TS3MockTest: public NUnitTest::TTestBase, private NWrappers::TS3User { - using TS3Mock = NWrappers::NTestHelpers::TS3Mock; - - static auto MakeClientConfig(ui16 port) { - Aws::Client::ClientConfiguration config; - - config.endpointOverride = Sprintf("localhost:%hu", port); - config.verifySSL = false; - config.connectTimeoutMs = 10000; - config.maxConnections = 5; - config.scheme = Aws::Http::Scheme::HTTP; - - return config; - } - -public: - void SetUp() override { - UNIT_ASSERT(!Port.Defined()); - Port = PortManager.GetPort(); - - S3Mock = MakeHolder<TS3Mock>(TS3Mock::TSettings(*Port)); - UNIT_ASSERT(S3Mock->Start()); - - Runtime = MakeHolder<TTestBasicRuntime>(); - Runtime->Initialize(TAppPrepare().Unwrap()); - Runtime->SetLogPriority(NKikimrServices::S3_WRAPPER, NLog::PRI_DEBUG); - Wrapper = Runtime->Register(NWrappers::CreateS3Wrapper(Aws::Auth::AWSCredentials(), MakeClientConfig(*Port))); - } - - void TearDown() override { - S3Mock.Reset(); - Runtime.Reset(); - } - - ui16 GetPort() const { - UNIT_ASSERT(Port.Defined()); - return *Port; - } - - template <typename TEvResponse> - auto Send(IEventBase* ev) { - if (!Edge) { - Edge = Runtime->AllocateEdgeActor(); - } - - Runtime->Send(new IEventHandle(Wrapper, *Edge, ev)); - return Runtime->GrabEdgeEvent<TEvResponse>(*Edge); - } - -private: - TPortManager PortManager; - TMaybe<ui16> Port; - THolder<TS3Mock> S3Mock; - THolder<TTestBasicRuntime> Runtime; - TActorId Wrapper; - TMaybe<TActorId> Edge; - -}; // TS3MockTest - -class TS3WrapperTests: public TS3MockTest { - using TEvS3Wrapper = NWrappers::TEvS3Wrapper; - - auto PutObject(const TString& key, TString&& body) { - auto request = PutObjectRequest().WithBucket("").WithKey(key); - auto response = Send<TEvS3Wrapper::TEvPutObjectResponse>( - new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(body))); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - - auto HeadObject(const TString& key) { - auto request = HeadObjectRequest().WithBucket("").WithKey(key); - auto response = Send<TEvS3Wrapper::TEvHeadObjectResponse>( - new TEvS3Wrapper::TEvHeadObjectRequest(request)); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - - auto GetObject(const TString& key, ui64 bodySize) { - auto request = GetObjectRequest() - .WithBucket("") - .WithKey(key) - .WithRange(Sprintf("bytes=0-%" PRIu64, bodySize - 1)); - auto response = Send<TEvS3Wrapper::TEvGetObjectResponse>( - new TEvS3Wrapper::TEvGetObjectRequest(request)); - - UNIT_ASSERT(response->Get()); - return std::make_pair(response->Get()->Result, response->Get()->Body); - } - - auto CreateMultipartUpload(const TString& key) { - auto request = CreateMultipartUploadRequest().WithBucket("").WithKey(key); - auto response = Send<TEvS3Wrapper::TEvCreateMultipartUploadResponse>( - new TEvS3Wrapper::TEvCreateMultipartUploadRequest(request)); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - - auto CompleteMultipartUpload(const TString& key, const TString& uploadId, const TVector<CompletedPart>& parts) { - auto request = CompleteMultipartUploadRequest() - .WithBucket("") - .WithKey(key) - .WithUploadId(uploadId) - .WithMultipartUpload(CompletedMultipartUpload().WithParts(parts)); - auto response = Send<TEvS3Wrapper::TEvCompleteMultipartUploadResponse>( - new TEvS3Wrapper::TEvCompleteMultipartUploadRequest(request)); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - - auto AbortMultipartUpload(const TString& key, const TString& uploadId) { - auto request = AbortMultipartUploadRequest() - .WithBucket("") - .WithKey(key) - .WithUploadId(uploadId); - auto response = Send<TEvS3Wrapper::TEvAbortMultipartUploadResponse>( - new TEvS3Wrapper::TEvAbortMultipartUploadRequest(request)); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - - auto UploadPart(const TString& key, const TString& uploadId, int partNumber, TString&& body) { - auto request = UploadPartRequest() - .WithBucket("") - .WithKey(key) - .WithUploadId(uploadId) - .WithPartNumber(partNumber); - auto response = Send<TEvS3Wrapper::TEvUploadPartResponse>( - new TEvS3Wrapper::TEvUploadPartRequest(request, std::move(body))); - - UNIT_ASSERT(response->Get()); - return response->Get()->Result; - } - -public: - void PutObject() { - auto result = PutObject("key", "body"); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - } - - void HeadObject() { - const TString body = "body"; - - { - auto result = PutObject("key", TString(body)); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - } - - { - auto result = HeadObject("key"); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().GetETag(), MD5::Data(body)); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().GetContentLength(), body.size()); - } - } - - void GetObject() { - const TString body = "body"; - - { - auto result = PutObject("key", TString(body)); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - } - - { - auto [result, actualBody] = GetObject("key", body.size()); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - UNIT_ASSERT_VALUES_EQUAL(actualBody, body); - } - } - - void MultipartUpload() { - const TString body = "body"; - TString uploadId; - TVector<CompletedPart> parts; - - { - auto result = CreateMultipartUpload("key"); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - uploadId = result.GetResult().GetUploadId(); - } - - { - auto result = UploadPart("key", uploadId, 1, TString(body)); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - parts.push_back(CompletedPart().WithPartNumber(1).WithETag(result.GetResult().GetETag())); - } - - { - auto result = CompleteMultipartUpload("key", uploadId, parts); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - } - - { - auto [result, actualBody] = GetObject("key", body.size()); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - UNIT_ASSERT_VALUES_EQUAL(actualBody, body); - } - } - - void AbortMultipartUpload() { - TString uploadId; - - { - auto result = CreateMultipartUpload("key"); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - uploadId = result.GetResult().GetUploadId(); - } - - { - auto result = AbortMultipartUpload("key", uploadId); - UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); - } - - { - auto result = HeadObject("key"); - UNIT_ASSERT(!result.IsSuccess()); - } - } - - void HeadUnknownObject() { - auto result = HeadObject("key"); - UNIT_ASSERT(!result.IsSuccess()); - } - - void GetUnknownObject() { - auto [result, actualBody] = GetObject("key", 4); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_VALUES_EQUAL(actualBody, ""); - } - - void UploadUnknownPart() { - auto result = UploadPart("key", "uploadId", 1, "body"); - UNIT_ASSERT(!result.IsSuccess()); - } - - void CompleteUnknownUpload() { - auto result = CompleteMultipartUpload("key", "uploadId", { - CompletedPart().WithPartNumber(1).WithETag("ETag"), - }); - UNIT_ASSERT(!result.IsSuccess()); - } - - void AbortUnknownUpload() { - auto result = AbortMultipartUpload("key", "uploadId"); - UNIT_ASSERT(!result.IsSuccess()); - } - -private: - UNIT_TEST_SUITE(TS3WrapperTests); - UNIT_TEST(PutObject); - UNIT_TEST(HeadObject); - UNIT_TEST(GetObject); - UNIT_TEST(MultipartUpload); - UNIT_TEST(AbortMultipartUpload); - UNIT_TEST(HeadUnknownObject); - UNIT_TEST(GetUnknownObject); - UNIT_TEST(UploadUnknownPart); - UNIT_TEST(CompleteUnknownUpload); - UNIT_TEST(AbortUnknownUpload); - UNIT_TEST_SUITE_END(); - -}; // TS3WrapperTests - -UNIT_TEST_SUITE_REGISTRATION(TS3WrapperTests); + +#include <library/cpp/actors/core/log.h> +#include <library/cpp/digest/md5/md5.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/printf.h> + +using namespace NActors; +using namespace NKikimr; +using namespace Aws::S3::Model; + +class TS3MockTest: public NUnitTest::TTestBase, private NWrappers::TS3User { + using TS3Mock = NWrappers::NTestHelpers::TS3Mock; + + static auto MakeClientConfig(ui16 port) { + Aws::Client::ClientConfiguration config; + + config.endpointOverride = Sprintf("localhost:%hu", port); + config.verifySSL = false; + config.connectTimeoutMs = 10000; + config.maxConnections = 5; + config.scheme = Aws::Http::Scheme::HTTP; + + return config; + } + +public: + void SetUp() override { + UNIT_ASSERT(!Port.Defined()); + Port = PortManager.GetPort(); + + S3Mock = MakeHolder<TS3Mock>(TS3Mock::TSettings(*Port)); + UNIT_ASSERT(S3Mock->Start()); + + Runtime = MakeHolder<TTestBasicRuntime>(); + Runtime->Initialize(TAppPrepare().Unwrap()); + Runtime->SetLogPriority(NKikimrServices::S3_WRAPPER, NLog::PRI_DEBUG); + Wrapper = Runtime->Register(NWrappers::CreateS3Wrapper(Aws::Auth::AWSCredentials(), MakeClientConfig(*Port))); + } + + void TearDown() override { + S3Mock.Reset(); + Runtime.Reset(); + } + + ui16 GetPort() const { + UNIT_ASSERT(Port.Defined()); + return *Port; + } + + template <typename TEvResponse> + auto Send(IEventBase* ev) { + if (!Edge) { + Edge = Runtime->AllocateEdgeActor(); + } + + Runtime->Send(new IEventHandle(Wrapper, *Edge, ev)); + return Runtime->GrabEdgeEvent<TEvResponse>(*Edge); + } + +private: + TPortManager PortManager; + TMaybe<ui16> Port; + THolder<TS3Mock> S3Mock; + THolder<TTestBasicRuntime> Runtime; + TActorId Wrapper; + TMaybe<TActorId> Edge; + +}; // TS3MockTest + +class TS3WrapperTests: public TS3MockTest { + using TEvS3Wrapper = NWrappers::TEvS3Wrapper; + + auto PutObject(const TString& key, TString&& body) { + auto request = PutObjectRequest().WithBucket("").WithKey(key); + auto response = Send<TEvS3Wrapper::TEvPutObjectResponse>( + new TEvS3Wrapper::TEvPutObjectRequest(request, std::move(body))); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + + auto HeadObject(const TString& key) { + auto request = HeadObjectRequest().WithBucket("").WithKey(key); + auto response = Send<TEvS3Wrapper::TEvHeadObjectResponse>( + new TEvS3Wrapper::TEvHeadObjectRequest(request)); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + + auto GetObject(const TString& key, ui64 bodySize) { + auto request = GetObjectRequest() + .WithBucket("") + .WithKey(key) + .WithRange(Sprintf("bytes=0-%" PRIu64, bodySize - 1)); + auto response = Send<TEvS3Wrapper::TEvGetObjectResponse>( + new TEvS3Wrapper::TEvGetObjectRequest(request)); + + UNIT_ASSERT(response->Get()); + return std::make_pair(response->Get()->Result, response->Get()->Body); + } + + auto CreateMultipartUpload(const TString& key) { + auto request = CreateMultipartUploadRequest().WithBucket("").WithKey(key); + auto response = Send<TEvS3Wrapper::TEvCreateMultipartUploadResponse>( + new TEvS3Wrapper::TEvCreateMultipartUploadRequest(request)); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + + auto CompleteMultipartUpload(const TString& key, const TString& uploadId, const TVector<CompletedPart>& parts) { + auto request = CompleteMultipartUploadRequest() + .WithBucket("") + .WithKey(key) + .WithUploadId(uploadId) + .WithMultipartUpload(CompletedMultipartUpload().WithParts(parts)); + auto response = Send<TEvS3Wrapper::TEvCompleteMultipartUploadResponse>( + new TEvS3Wrapper::TEvCompleteMultipartUploadRequest(request)); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + + auto AbortMultipartUpload(const TString& key, const TString& uploadId) { + auto request = AbortMultipartUploadRequest() + .WithBucket("") + .WithKey(key) + .WithUploadId(uploadId); + auto response = Send<TEvS3Wrapper::TEvAbortMultipartUploadResponse>( + new TEvS3Wrapper::TEvAbortMultipartUploadRequest(request)); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + + auto UploadPart(const TString& key, const TString& uploadId, int partNumber, TString&& body) { + auto request = UploadPartRequest() + .WithBucket("") + .WithKey(key) + .WithUploadId(uploadId) + .WithPartNumber(partNumber); + auto response = Send<TEvS3Wrapper::TEvUploadPartResponse>( + new TEvS3Wrapper::TEvUploadPartRequest(request, std::move(body))); + + UNIT_ASSERT(response->Get()); + return response->Get()->Result; + } + +public: + void PutObject() { + auto result = PutObject("key", "body"); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + } + + void HeadObject() { + const TString body = "body"; + + { + auto result = PutObject("key", TString(body)); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + } + + { + auto result = HeadObject("key"); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().GetETag(), MD5::Data(body)); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().GetContentLength(), body.size()); + } + } + + void GetObject() { + const TString body = "body"; + + { + auto result = PutObject("key", TString(body)); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + } + + { + auto [result, actualBody] = GetObject("key", body.size()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL(actualBody, body); + } + } + + void MultipartUpload() { + const TString body = "body"; + TString uploadId; + TVector<CompletedPart> parts; + + { + auto result = CreateMultipartUpload("key"); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + uploadId = result.GetResult().GetUploadId(); + } + + { + auto result = UploadPart("key", uploadId, 1, TString(body)); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + parts.push_back(CompletedPart().WithPartNumber(1).WithETag(result.GetResult().GetETag())); + } + + { + auto result = CompleteMultipartUpload("key", uploadId, parts); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + } + + { + auto [result, actualBody] = GetObject("key", body.size()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL(actualBody, body); + } + } + + void AbortMultipartUpload() { + TString uploadId; + + { + auto result = CreateMultipartUpload("key"); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + uploadId = result.GetResult().GetUploadId(); + } + + { + auto result = AbortMultipartUpload("key", uploadId); + UNIT_ASSERT_C(result.IsSuccess(), result.GetError().GetMessage()); + } + + { + auto result = HeadObject("key"); + UNIT_ASSERT(!result.IsSuccess()); + } + } + + void HeadUnknownObject() { + auto result = HeadObject("key"); + UNIT_ASSERT(!result.IsSuccess()); + } + + void GetUnknownObject() { + auto [result, actualBody] = GetObject("key", 4); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(actualBody, ""); + } + + void UploadUnknownPart() { + auto result = UploadPart("key", "uploadId", 1, "body"); + UNIT_ASSERT(!result.IsSuccess()); + } + + void CompleteUnknownUpload() { + auto result = CompleteMultipartUpload("key", "uploadId", { + CompletedPart().WithPartNumber(1).WithETag("ETag"), + }); + UNIT_ASSERT(!result.IsSuccess()); + } + + void AbortUnknownUpload() { + auto result = AbortMultipartUpload("key", "uploadId"); + UNIT_ASSERT(!result.IsSuccess()); + } + +private: + UNIT_TEST_SUITE(TS3WrapperTests); + UNIT_TEST(PutObject); + UNIT_TEST(HeadObject); + UNIT_TEST(GetObject); + UNIT_TEST(MultipartUpload); + UNIT_TEST(AbortMultipartUpload); + UNIT_TEST(HeadUnknownObject); + UNIT_TEST(GetUnknownObject); + UNIT_TEST(UploadUnknownPart); + UNIT_TEST(CompleteUnknownUpload); + UNIT_TEST(AbortUnknownUpload); + UNIT_TEST_SUITE_END(); + +}; // TS3WrapperTests + +UNIT_TEST_SUITE_REGISTRATION(TS3WrapperTests); diff --git a/ydb/core/wrappers/ut/ya.make b/ydb/core/wrappers/ut/ya.make index d88cea70ab1..cbc8f4b3533 100644 --- a/ydb/core/wrappers/ut/ya.make +++ b/ydb/core/wrappers/ut/ya.make @@ -1,28 +1,28 @@ UNITTEST_FOR(ydb/core/wrappers) - -OWNER( - ilnaz - g:kikimr -) - -FORK_SUBTESTS() - -IF (NOT OS_WINDOWS) - PEERDIR( - library/cpp/actors/core - library/cpp/digest/md5 - library/cpp/testing/unittest + +OWNER( + ilnaz + g:kikimr +) + +FORK_SUBTESTS() + +IF (NOT OS_WINDOWS) + PEERDIR( + library/cpp/actors/core + library/cpp/digest/md5 + library/cpp/testing/unittest ydb/core/protos ydb/core/testlib/basics ydb/core/wrappers/ut_helpers - ) - SRCS( - s3_wrapper_ut.cpp - ) -ENDIF() - + ) + SRCS( + s3_wrapper_ut.cpp + ) +ENDIF() + YQL_LAST_ABI_VERSION() - + REQUIREMENTS(ram:12) -END() +END() diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.cpp b/ydb/core/wrappers/ut_helpers/s3_mock.cpp index a1211dea93a..8df5acc2001 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.cpp +++ b/ydb/core/wrappers/ut_helpers/s3_mock.cpp @@ -1,334 +1,334 @@ -#include "s3_mock.h" - -#include <library/cpp/digest/md5/md5.h> -#include <library/cpp/xml/document/xml-document.h> - -#include <util/generic/xrange.h> -#include <util/random/shuffle.h> -#include <util/string/cast.h> -#include <util/string/join.h> -#include <util/string/printf.h> - -namespace NKikimr { -namespace NWrappers { -namespace NTestHelpers { - -TS3Mock::TSettings::TSettings() - : CorruptETags(false) - , RejectUploadParts(false) -{ -} - -TS3Mock::TSettings::TSettings(ui16 port) - : HttpOptions(TOptions(port).SetThreads(1)) - , CorruptETags(false) - , RejectUploadParts(false) -{ -} - -TS3Mock::TSettings& TS3Mock::TSettings::WithHttpOptions(const TOptions& opts) { - HttpOptions = opts; - return *this; -} - -TS3Mock::TSettings& TS3Mock::TSettings::WithCorruptETags(bool value) { - CorruptETags = value; - return *this; -} - -TS3Mock::TSettings& TS3Mock::TSettings::WithRejectUploadParts(bool value) { - RejectUploadParts = value; - return *this; -} - -TS3Mock::TRequest::EMethod TS3Mock::TRequest::ParseMethod(const char* str) { - if (strnicmp(str, "HEAD", 4) == 0) { - return EMethod::Head; - } else if (strnicmp(str, "GET", 3) == 0) { - return EMethod::Get; - } else if (strnicmp(str, "PUT", 3) == 0) { - return EMethod::Put; - } else if (strnicmp(str, "POST", 4) == 0) { - return EMethod::Post; - } else if (strnicmp(str, "PATCH", 5) == 0) { - return EMethod::Patch; - } else if (strnicmp(str, "DELETE", 6) == 0) { - return EMethod::Delete; - } - return EMethod::NotImplemented; -} - -bool TS3Mock::TRequest::TryParseRange(TStringBuf str, std::pair<ui32, ui32>& range) { - if (!str.SkipPrefix("bytes=")) { - return false; - } - - ui32 start; - if (!TryFromString(str.NextTok('-'), start)) { - return false; - } - - ui32 end; - if (!TryFromString(str, end)) { - return false; - } - - range = std::make_pair(start, end); - return true; -} - -bool TS3Mock::TRequest::HttpBadRequest(const TReplyParams& params, const TString& error) { - params.Output << "HTTP/1.1 400 Bad request\r\n\r\n"; - params.Output << error; - return true; -} - -bool TS3Mock::TRequest::HttpNotFound(const TReplyParams& params) { - params.Output << "HTTP/1.1 404 Not found\r\n\r\n"; - return true; -} - -bool TS3Mock::TRequest::HttpNotImplemented(const TReplyParams& params) { - params.Output << "HTTP/1.1 501 Not Implemented\r\n\r\n"; - return true; -} - -void TS3Mock::TRequest::MaybeContinue(const TReplyParams& params) { - if (params.Input.HasExpect100Continue()) { - params.Output.SendContinue(); - } -} - -bool TS3Mock::TRequest::HttpServeRead(const TReplyParams& params, EMethod method, TStringBuf content) { - std::pair<ui32, ui32> range(0, content.size() - 1); - if (const auto* rangeHeader = params.Input.Headers().FindHeader("Range")) { - if (!TryParseRange(rangeHeader->Value(), range)) { - return HttpBadRequest(params, "Invalid range"); - } - } - - TString etag = MD5::Data(content); - if (Parent->Settings.CorruptETags) { - ShuffleRange(etag); - } - - params.Output << "HTTP/1.1 200 Ok\r\n"; - THttpHeaders headers; - headers.AddHeader("Content-Length", range.second - range.first + 1); - headers.AddHeader("ETag", etag); - headers.OutTo(¶ms.Output); - params.Output << "\r\n"; - - if (method == EMethod::Get) { - params.Output << content.SubStr(range.first, range.second - range.first + 1); - } - - return true; -} - -bool TS3Mock::TRequest::HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams) { - TString content; - ui64 length; - - if (params.Input.GetContentLength(length)) { - content = TString::Uninitialized(length); - params.Input.Read(content.Detach(), length); - } else { - content = params.Input.ReadAll(); - } - - const TString etag = MD5::Data(content); - - if (!queryParams) { - Parent->Data[path] = std::move(content); - } else { - if (!queryParams.Has("uploadId") || !queryParams.Has("partNumber")) { - return HttpBadRequest(params, "'uploadId' & 'partNumber' must be specified"); - } - - auto it = Parent->MultipartUploads.find(std::make_pair(path, queryParams.Get("uploadId"))); - if (it == Parent->MultipartUploads.end()) { - return HttpBadRequest(params, "Invalid uploadId"); - } - - size_t partNumber = 0; - if (!TryFromString(queryParams.Get("partNumber"), partNumber) || !partNumber) { - return HttpBadRequest(params, "Invalid partNumber"); - } - - if (Parent->Settings.RejectUploadParts) { - return HttpBadRequest(params, "Reject"); - } - - auto& parts = it->second; - if (parts.size() < partNumber) { - parts.resize(partNumber); - } - - parts[partNumber - 1] = std::move(content); - } - - params.Output << "HTTP/1.1 200 Ok\r\n"; - THttpHeaders headers; - headers.AddHeader("ETag", etag); - headers.OutTo(¶ms.Output); - params.Output << "\r\n"; - - return true; -} - -bool TS3Mock::TRequest::HttpServeAction(const TReplyParams& params, EMethod method, TStringBuf path, const TCgiParameters& queryParams) { - if (queryParams.Has("uploads")) { - const int uploadId = Parent->NextUploadId++; - Parent->MultipartUploads[std::make_pair(path, ToString(uploadId))] = {}; - - params.Output << "HTTP/1.1 200 Ok\r\n"; - params.Output << Sprintf(R"( - <?xml version="1.0" encoding="UTF-8"?> - <InitiateMultipartUploadResult> - <Key>%s</Key> - <UploadId>%i</UploadId> - </InitiateMultipartUploadResult> - )", TString(path.After('/')).c_str(), uploadId); - } else if (queryParams.Has("uploadId")) { - auto it = Parent->MultipartUploads.find(std::make_pair(path, queryParams.Get("uploadId"))); - if (it == Parent->MultipartUploads.end()) { - return HttpBadRequest(params, "Invalid uploadId"); - } - - if (method == EMethod::Post) { - NXml::TDocument request(params.Input.ReadAll(), NXml::TDocument::String); - TVector<TString> etags; - - auto part = request.Root().FirstChild("Part"); - while (!part.IsNull()) { - auto partNumberNode = part.FirstChild("PartNumber"); - if (partNumberNode.IsNull()) { - return HttpBadRequest(params, "Invalid request"); - } - - const ui32 partNumber = partNumberNode.Value<ui32>(); - if (etags.size() < partNumber) { - etags.resize(partNumber); - } - - auto etagNode = part.FirstChild("ETag"); - if (etagNode.IsNull()) { - return HttpBadRequest(params, "Invalid request"); - } - - etags[partNumber - 1] = etagNode.Value<TString>(); - part = part.NextSibling("Part"); - } - - if (etags.size() != it->second.size()) { - return HttpBadRequest(params, "Invalid part count"); - } - - for (auto i : xrange(etags.size())) { - if (etags.at(i) != MD5::Data(it->second.at(i))) { - return HttpBadRequest(params, "Invalid part"); - } - } - - Parent->Data[path] = JoinSeq("", it->second); - Parent->MultipartUploads.erase(it); - - const TString etag = MD5::Data(Parent->Data[path]); - - params.Output << "HTTP/1.1 200 Ok\r\n"; - params.Output << Sprintf(R"( - <?xml version="1.0" encoding="UTF-8"?> - <CompleteMultipartUploadResult> - <Key>%s</Key> - <ETag>%s</ETag> - </CompleteMultipartUploadResult> - )", TString(path.After('/')).c_str(), etag.c_str()); - } else if (method == EMethod::Delete) { - Parent->MultipartUploads.erase(it); - params.Output << "HTTP/1.1 204 Ok\r\n\r\n"; - } else { - return HttpBadRequest(params); - } - } else { - return HttpBadRequest(params); - } - - return true; -} - -TS3Mock::TRequest::TRequest(TS3Mock* parent) - : Parent(parent) -{ -} - -bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { - TStringBuf requestString(RequestString); - TStringBuf methodStr = requestString.NextTok(' '); - TStringBuf uriStr = requestString.NextTok(' '); - - if (!methodStr.IsInited() || !uriStr.IsInited()) { - return HttpBadRequest(params); - } - - const EMethod method = ParseMethod(methodStr.data()); - TStringBuf pathStr = uriStr.NextTok('?'); - - TCgiParameters queryParams; - queryParams.ScanAddAll(uriStr); - - switch (method) { - case EMethod::NotImplemented: - return HttpNotImplemented(params); - - case EMethod::Head: - case EMethod::Get: - if (Parent->Data.contains(pathStr)) { - return HttpServeRead(params, method, Parent->Data.at(pathStr)); - } else { - return HttpNotFound(params); - } - break; - - case EMethod::Put: - MaybeContinue(params); - return HttpServeWrite(params, pathStr, queryParams); - - case EMethod::Post: - case EMethod::Delete: - MaybeContinue(params); - return HttpServeAction(params, method, pathStr, queryParams); - - case EMethod::Patch: - MaybeContinue(params); - return HttpNotImplemented(params); - } -} - -TS3Mock::TS3Mock(const TSettings& settings) - : THttpServer(this, settings.HttpOptions) - , Settings(settings) -{ -} - -TS3Mock::TS3Mock(THashMap<TString, TString>&& data, const TSettings& settings) - : THttpServer(this, settings.HttpOptions) - , Settings(settings) - , Data(std::move(data)) -{ -} - -TS3Mock::TS3Mock(const THashMap<TString, TString>& data, const TSettings& settings) - : THttpServer(this, settings.HttpOptions) - , Settings(settings) - , Data(data) -{ -} - -TClientRequest* TS3Mock::CreateClient() { - return new TRequest(this); -} - -} // NTestHelpers -} // NWrappers -} // NKikimr +#include "s3_mock.h" + +#include <library/cpp/digest/md5/md5.h> +#include <library/cpp/xml/document/xml-document.h> + +#include <util/generic/xrange.h> +#include <util/random/shuffle.h> +#include <util/string/cast.h> +#include <util/string/join.h> +#include <util/string/printf.h> + +namespace NKikimr { +namespace NWrappers { +namespace NTestHelpers { + +TS3Mock::TSettings::TSettings() + : CorruptETags(false) + , RejectUploadParts(false) +{ +} + +TS3Mock::TSettings::TSettings(ui16 port) + : HttpOptions(TOptions(port).SetThreads(1)) + , CorruptETags(false) + , RejectUploadParts(false) +{ +} + +TS3Mock::TSettings& TS3Mock::TSettings::WithHttpOptions(const TOptions& opts) { + HttpOptions = opts; + return *this; +} + +TS3Mock::TSettings& TS3Mock::TSettings::WithCorruptETags(bool value) { + CorruptETags = value; + return *this; +} + +TS3Mock::TSettings& TS3Mock::TSettings::WithRejectUploadParts(bool value) { + RejectUploadParts = value; + return *this; +} + +TS3Mock::TRequest::EMethod TS3Mock::TRequest::ParseMethod(const char* str) { + if (strnicmp(str, "HEAD", 4) == 0) { + return EMethod::Head; + } else if (strnicmp(str, "GET", 3) == 0) { + return EMethod::Get; + } else if (strnicmp(str, "PUT", 3) == 0) { + return EMethod::Put; + } else if (strnicmp(str, "POST", 4) == 0) { + return EMethod::Post; + } else if (strnicmp(str, "PATCH", 5) == 0) { + return EMethod::Patch; + } else if (strnicmp(str, "DELETE", 6) == 0) { + return EMethod::Delete; + } + return EMethod::NotImplemented; +} + +bool TS3Mock::TRequest::TryParseRange(TStringBuf str, std::pair<ui32, ui32>& range) { + if (!str.SkipPrefix("bytes=")) { + return false; + } + + ui32 start; + if (!TryFromString(str.NextTok('-'), start)) { + return false; + } + + ui32 end; + if (!TryFromString(str, end)) { + return false; + } + + range = std::make_pair(start, end); + return true; +} + +bool TS3Mock::TRequest::HttpBadRequest(const TReplyParams& params, const TString& error) { + params.Output << "HTTP/1.1 400 Bad request\r\n\r\n"; + params.Output << error; + return true; +} + +bool TS3Mock::TRequest::HttpNotFound(const TReplyParams& params) { + params.Output << "HTTP/1.1 404 Not found\r\n\r\n"; + return true; +} + +bool TS3Mock::TRequest::HttpNotImplemented(const TReplyParams& params) { + params.Output << "HTTP/1.1 501 Not Implemented\r\n\r\n"; + return true; +} + +void TS3Mock::TRequest::MaybeContinue(const TReplyParams& params) { + if (params.Input.HasExpect100Continue()) { + params.Output.SendContinue(); + } +} + +bool TS3Mock::TRequest::HttpServeRead(const TReplyParams& params, EMethod method, TStringBuf content) { + std::pair<ui32, ui32> range(0, content.size() - 1); + if (const auto* rangeHeader = params.Input.Headers().FindHeader("Range")) { + if (!TryParseRange(rangeHeader->Value(), range)) { + return HttpBadRequest(params, "Invalid range"); + } + } + + TString etag = MD5::Data(content); + if (Parent->Settings.CorruptETags) { + ShuffleRange(etag); + } + + params.Output << "HTTP/1.1 200 Ok\r\n"; + THttpHeaders headers; + headers.AddHeader("Content-Length", range.second - range.first + 1); + headers.AddHeader("ETag", etag); + headers.OutTo(¶ms.Output); + params.Output << "\r\n"; + + if (method == EMethod::Get) { + params.Output << content.SubStr(range.first, range.second - range.first + 1); + } + + return true; +} + +bool TS3Mock::TRequest::HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams) { + TString content; + ui64 length; + + if (params.Input.GetContentLength(length)) { + content = TString::Uninitialized(length); + params.Input.Read(content.Detach(), length); + } else { + content = params.Input.ReadAll(); + } + + const TString etag = MD5::Data(content); + + if (!queryParams) { + Parent->Data[path] = std::move(content); + } else { + if (!queryParams.Has("uploadId") || !queryParams.Has("partNumber")) { + return HttpBadRequest(params, "'uploadId' & 'partNumber' must be specified"); + } + + auto it = Parent->MultipartUploads.find(std::make_pair(path, queryParams.Get("uploadId"))); + if (it == Parent->MultipartUploads.end()) { + return HttpBadRequest(params, "Invalid uploadId"); + } + + size_t partNumber = 0; + if (!TryFromString(queryParams.Get("partNumber"), partNumber) || !partNumber) { + return HttpBadRequest(params, "Invalid partNumber"); + } + + if (Parent->Settings.RejectUploadParts) { + return HttpBadRequest(params, "Reject"); + } + + auto& parts = it->second; + if (parts.size() < partNumber) { + parts.resize(partNumber); + } + + parts[partNumber - 1] = std::move(content); + } + + params.Output << "HTTP/1.1 200 Ok\r\n"; + THttpHeaders headers; + headers.AddHeader("ETag", etag); + headers.OutTo(¶ms.Output); + params.Output << "\r\n"; + + return true; +} + +bool TS3Mock::TRequest::HttpServeAction(const TReplyParams& params, EMethod method, TStringBuf path, const TCgiParameters& queryParams) { + if (queryParams.Has("uploads")) { + const int uploadId = Parent->NextUploadId++; + Parent->MultipartUploads[std::make_pair(path, ToString(uploadId))] = {}; + + params.Output << "HTTP/1.1 200 Ok\r\n"; + params.Output << Sprintf(R"( + <?xml version="1.0" encoding="UTF-8"?> + <InitiateMultipartUploadResult> + <Key>%s</Key> + <UploadId>%i</UploadId> + </InitiateMultipartUploadResult> + )", TString(path.After('/')).c_str(), uploadId); + } else if (queryParams.Has("uploadId")) { + auto it = Parent->MultipartUploads.find(std::make_pair(path, queryParams.Get("uploadId"))); + if (it == Parent->MultipartUploads.end()) { + return HttpBadRequest(params, "Invalid uploadId"); + } + + if (method == EMethod::Post) { + NXml::TDocument request(params.Input.ReadAll(), NXml::TDocument::String); + TVector<TString> etags; + + auto part = request.Root().FirstChild("Part"); + while (!part.IsNull()) { + auto partNumberNode = part.FirstChild("PartNumber"); + if (partNumberNode.IsNull()) { + return HttpBadRequest(params, "Invalid request"); + } + + const ui32 partNumber = partNumberNode.Value<ui32>(); + if (etags.size() < partNumber) { + etags.resize(partNumber); + } + + auto etagNode = part.FirstChild("ETag"); + if (etagNode.IsNull()) { + return HttpBadRequest(params, "Invalid request"); + } + + etags[partNumber - 1] = etagNode.Value<TString>(); + part = part.NextSibling("Part"); + } + + if (etags.size() != it->second.size()) { + return HttpBadRequest(params, "Invalid part count"); + } + + for (auto i : xrange(etags.size())) { + if (etags.at(i) != MD5::Data(it->second.at(i))) { + return HttpBadRequest(params, "Invalid part"); + } + } + + Parent->Data[path] = JoinSeq("", it->second); + Parent->MultipartUploads.erase(it); + + const TString etag = MD5::Data(Parent->Data[path]); + + params.Output << "HTTP/1.1 200 Ok\r\n"; + params.Output << Sprintf(R"( + <?xml version="1.0" encoding="UTF-8"?> + <CompleteMultipartUploadResult> + <Key>%s</Key> + <ETag>%s</ETag> + </CompleteMultipartUploadResult> + )", TString(path.After('/')).c_str(), etag.c_str()); + } else if (method == EMethod::Delete) { + Parent->MultipartUploads.erase(it); + params.Output << "HTTP/1.1 204 Ok\r\n\r\n"; + } else { + return HttpBadRequest(params); + } + } else { + return HttpBadRequest(params); + } + + return true; +} + +TS3Mock::TRequest::TRequest(TS3Mock* parent) + : Parent(parent) +{ +} + +bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { + TStringBuf requestString(RequestString); + TStringBuf methodStr = requestString.NextTok(' '); + TStringBuf uriStr = requestString.NextTok(' '); + + if (!methodStr.IsInited() || !uriStr.IsInited()) { + return HttpBadRequest(params); + } + + const EMethod method = ParseMethod(methodStr.data()); + TStringBuf pathStr = uriStr.NextTok('?'); + + TCgiParameters queryParams; + queryParams.ScanAddAll(uriStr); + + switch (method) { + case EMethod::NotImplemented: + return HttpNotImplemented(params); + + case EMethod::Head: + case EMethod::Get: + if (Parent->Data.contains(pathStr)) { + return HttpServeRead(params, method, Parent->Data.at(pathStr)); + } else { + return HttpNotFound(params); + } + break; + + case EMethod::Put: + MaybeContinue(params); + return HttpServeWrite(params, pathStr, queryParams); + + case EMethod::Post: + case EMethod::Delete: + MaybeContinue(params); + return HttpServeAction(params, method, pathStr, queryParams); + + case EMethod::Patch: + MaybeContinue(params); + return HttpNotImplemented(params); + } +} + +TS3Mock::TS3Mock(const TSettings& settings) + : THttpServer(this, settings.HttpOptions) + , Settings(settings) +{ +} + +TS3Mock::TS3Mock(THashMap<TString, TString>&& data, const TSettings& settings) + : THttpServer(this, settings.HttpOptions) + , Settings(settings) + , Data(std::move(data)) +{ +} + +TS3Mock::TS3Mock(const THashMap<TString, TString>& data, const TSettings& settings) + : THttpServer(this, settings.HttpOptions) + , Settings(settings) + , Data(data) +{ +} + +TClientRequest* TS3Mock::CreateClient() { + return new TRequest(this); +} + +} // NTestHelpers +} // NWrappers +} // NKikimr diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.h b/ydb/core/wrappers/ut_helpers/s3_mock.h index a3ff85efd80..4d47fec47f5 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.h +++ b/ydb/core/wrappers/ut_helpers/s3_mock.h @@ -1,81 +1,81 @@ -#pragma once - -#include <library/cpp/http/server/http.h> -#include <library/cpp/cgiparam/cgiparam.h> - -#include <util/generic/hash.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NKikimr { -namespace NWrappers { -namespace NTestHelpers { - -class TS3Mock: public THttpServer, public THttpServer::ICallBack { -public: - struct TSettings { - TOptions HttpOptions; - bool CorruptETags; - bool RejectUploadParts; - - TSettings(); - explicit TSettings(ui16 port); - - TSettings& WithHttpOptions(const TOptions& opts); - TSettings& WithCorruptETags(bool value); - TSettings& WithRejectUploadParts(bool value); - - }; // TSettings - -private: - class TRequest: public TRequestReplier { - enum class EMethod { - NotImplemented, - Head, - Get, - Put, - Post, - Patch, - Delete, - }; - - static EMethod ParseMethod(const char* str); - static bool TryParseRange(TStringBuf str, std::pair<ui32, ui32>& range); - - bool HttpBadRequest(const TReplyParams& params, const TString& error = {}); - bool HttpNotFound(const TReplyParams& params); - bool HttpNotImplemented(const TReplyParams& params); - void MaybeContinue(const TReplyParams& params); - bool HttpServeRead(const TReplyParams& params, EMethod method, TStringBuf content); - bool HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams); - bool HttpServeAction(const TReplyParams& params, EMethod method, TStringBuf path, const TCgiParameters& queryParams); - - public: - explicit TRequest(TS3Mock* parent); - - bool DoReply(const TReplyParams& params) override; - - private: - TS3Mock* const Parent; - - }; // TRequest - -public: - explicit TS3Mock(const TSettings& settings = {}); - explicit TS3Mock(THashMap<TString, TString>&& data, const TSettings& settings = {}); - explicit TS3Mock(const THashMap<TString, TString>& data, const TSettings& settings = {}); - - TClientRequest* CreateClient() override; - -private: - const TSettings Settings; - THashMap<TString, TString> Data; - - int NextUploadId = 1; - THashMap<std::pair<TString, TString>, TVector<TString>> MultipartUploads; - -}; // TS3Mock - -} // NTestHelpers -} // NWrappers -} // NKikimr +#pragma once + +#include <library/cpp/http/server/http.h> +#include <library/cpp/cgiparam/cgiparam.h> + +#include <util/generic/hash.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NKikimr { +namespace NWrappers { +namespace NTestHelpers { + +class TS3Mock: public THttpServer, public THttpServer::ICallBack { +public: + struct TSettings { + TOptions HttpOptions; + bool CorruptETags; + bool RejectUploadParts; + + TSettings(); + explicit TSettings(ui16 port); + + TSettings& WithHttpOptions(const TOptions& opts); + TSettings& WithCorruptETags(bool value); + TSettings& WithRejectUploadParts(bool value); + + }; // TSettings + +private: + class TRequest: public TRequestReplier { + enum class EMethod { + NotImplemented, + Head, + Get, + Put, + Post, + Patch, + Delete, + }; + + static EMethod ParseMethod(const char* str); + static bool TryParseRange(TStringBuf str, std::pair<ui32, ui32>& range); + + bool HttpBadRequest(const TReplyParams& params, const TString& error = {}); + bool HttpNotFound(const TReplyParams& params); + bool HttpNotImplemented(const TReplyParams& params); + void MaybeContinue(const TReplyParams& params); + bool HttpServeRead(const TReplyParams& params, EMethod method, TStringBuf content); + bool HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams); + bool HttpServeAction(const TReplyParams& params, EMethod method, TStringBuf path, const TCgiParameters& queryParams); + + public: + explicit TRequest(TS3Mock* parent); + + bool DoReply(const TReplyParams& params) override; + + private: + TS3Mock* const Parent; + + }; // TRequest + +public: + explicit TS3Mock(const TSettings& settings = {}); + explicit TS3Mock(THashMap<TString, TString>&& data, const TSettings& settings = {}); + explicit TS3Mock(const THashMap<TString, TString>& data, const TSettings& settings = {}); + + TClientRequest* CreateClient() override; + +private: + const TSettings Settings; + THashMap<TString, TString> Data; + + int NextUploadId = 1; + THashMap<std::pair<TString, TString>, TVector<TString>> MultipartUploads; + +}; // TS3Mock + +} // NTestHelpers +} // NWrappers +} // NKikimr diff --git a/ydb/core/wrappers/ut_helpers/ya.make b/ydb/core/wrappers/ut_helpers/ya.make index fc4fbfc1828..41dd61bc46d 100644 --- a/ydb/core/wrappers/ut_helpers/ya.make +++ b/ydb/core/wrappers/ut_helpers/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -PEERDIR( - library/cpp/digest/md5 - library/cpp/http/server - library/cpp/xml/document - library/cpp/cgiparam -) - -SRCS( - s3_mock.cpp -) - -END() +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +PEERDIR( + library/cpp/digest/md5 + library/cpp/http/server + library/cpp/xml/document + library/cpp/cgiparam +) + +SRCS( + s3_mock.cpp +) + +END() diff --git a/ydb/core/wrappers/ya.make b/ydb/core/wrappers/ya.make index 31829e32855..addea944466 100644 --- a/ydb/core/wrappers/ya.make +++ b/ydb/core/wrappers/ya.make @@ -1,30 +1,30 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + IF (OS_WINDOWS) CFLAGS( -DKIKIMR_DISABLE_S3_WRAPPER ) -ELSE() - SRCS( - s3_out.cpp - s3_wrapper.cpp - ) - PEERDIR( - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3 - contrib/libs/curl - library/cpp/actors/core +ELSE() + SRCS( + s3_out.cpp + s3_wrapper.cpp + ) + PEERDIR( + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3 + contrib/libs/curl + library/cpp/actors/core ydb/core/base ydb/core/protos ydb/core/wrappers/ut_helpers - ) -ENDIF() - -END() + ) +ENDIF() + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/ydb_convert/column_families.h b/ydb/core/ydb_convert/column_families.h index 909787cc7a2..152e0561bc3 100644 --- a/ydb/core/ydb_convert/column_families.h +++ b/ydb/core/ydb_convert/column_families.h @@ -5,7 +5,7 @@ #include <ydb/public/api/protos/ydb_table.pb.h> #include <util/generic/hash.h> -#include <util/string/builder.h> +#include <util/string/builder.h> namespace NKikimr { diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 870eae7b5ad..0d10bb8450b 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -1,15 +1,15 @@ -#include "column_families.h" -#include "table_description.h" -#include "table_settings.h" -#include "ydb_convert.h" +#include "column_families.h" +#include "table_description.h" +#include "table_settings.h" +#include "ydb_convert.h" #include <ydb/core/base/appdata.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/issue_id.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> -#include <util/generic/hash.h> - +#include <util/generic/hash.h> + namespace NKikimr { static NProtoBuf::Timestamp MillisecToProtoTimeStamp(ui64 ms) { @@ -19,8 +19,8 @@ static NProtoBuf::Timestamp MillisecToProtoTimeStamp(ui64 ms) { return timestamp; } -template <typename TYdbProto> -void FillColumnDescriptionImpl(TYdbProto& out, +template <typename TYdbProto> +void FillColumnDescriptionImpl(TYdbProto& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { splitKeyType.SetKind(NKikimrMiniKQL::ETypeKind::Tuple); @@ -62,45 +62,45 @@ void FillColumnDescriptionImpl(TYdbProto& out, newColumn->set_family(column.GetFamilyName()); } } - - if (in.HasTTLSettings() && in.GetTTLSettings().HasEnabled()) { - const auto& inTTL = in.GetTTLSettings().GetEnabled(); - - switch (inTTL.GetColumnUnit()) { + + if (in.HasTTLSettings() && in.GetTTLSettings().HasEnabled()) { + const auto& inTTL = in.GetTTLSettings().GetEnabled(); + + switch (inTTL.GetColumnUnit()) { case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: { - auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - + auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column(); + outTTL.set_column_name(inTTL.GetColumnName()); + outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); + break; + } + case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: { - auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit())); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - - default: - break; - } - } -} - -void FillColumnDescription(Ydb::Table::DescribeTableResult& out, + auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch(); + outTTL.set_column_name(inTTL.GetColumnName()); + outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit())); + outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); + break; + } + + default: + break; + } + } +} + +void FillColumnDescription(Ydb::Table::DescribeTableResult& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { - FillColumnDescriptionImpl(out, splitKeyType, in); -} + FillColumnDescriptionImpl(out, splitKeyType, in); +} -void FillColumnDescription(Ydb::Table::CreateTableRequest& out, +void FillColumnDescription(Ydb::Table::CreateTableRequest& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { - FillColumnDescriptionImpl(out, splitKeyType, in); -} - + FillColumnDescriptionImpl(out, splitKeyType, in); +} + bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) { ui32 typeId; auto itemType = inType.has_optional_type() ? inType.optional_type().item() : inType; @@ -148,53 +148,53 @@ bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusId bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error) { - + for (const auto& column : in) { NKikimrSchemeOp:: TColumnDescription* cd = out.AddColumns(); - cd->SetName(column.name()); - if (!column.type().has_optional_type()) { - if (!AppData()->FeatureFlags.GetEnableNotNullColumns()) { + cd->SetName(column.name()); + if (!column.type().has_optional_type()) { + if (!AppData()->FeatureFlags.GetEnableNotNullColumns()) { status = Ydb::StatusIds::UNSUPPORTED; error = "Not null columns feature is not supported yet"; return false; } cd->SetNotNull(true); - } - - ui32 typeId; + } + + ui32 typeId; if (!ExtractColumnTypeId(typeId, column.type(), status, error)) { - return false; - } - cd->SetType(NYql::NProto::TypeIds_Name(NYql::NProto::TypeIds(typeId))); - - if (!column.family().empty()) { - cd->SetFamilyName(column.family()); - } - } - - return true; -} - -template <typename TYdbProto> -void FillTableBoundaryImpl(TYdbProto& out, + return false; + } + cd->SetType(NYql::NProto::TypeIds_Name(NYql::NProto::TypeIds(typeId))); + + if (!column.family().empty()) { + cd->SetFamilyName(column.family()); + } + } + + return true; +} + +template <typename TYdbProto> +void FillTableBoundaryImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { - + for (const auto& boundary : in.GetSplitBoundary()) { if (boundary.HasSerializedKeyPrefix()) { throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) << "Unexpected serialized response from txProxy"; } else if (boundary.HasKeyPrefix()) { - Ydb::TypedValue* ydbValue = nullptr; - - if constexpr (std::is_same<TYdbProto, Ydb::Table::DescribeTableResult>::value) { - ydbValue = out.add_shard_key_bounds(); - } else if constexpr (std::is_same<TYdbProto, Ydb::Table::CreateTableRequest>::value) { - ydbValue = out.mutable_partition_at_keys()->add_split_points(); - } else { - Y_FAIL("Unknown proto type"); - } - + Ydb::TypedValue* ydbValue = nullptr; + + if constexpr (std::is_same<TYdbProto, Ydb::Table::DescribeTableResult>::value) { + ydbValue = out.add_shard_key_bounds(); + } else if constexpr (std::is_same<TYdbProto, Ydb::Table::CreateTableRequest>::value) { + ydbValue = out.mutable_partition_at_keys()->add_split_points(); + } else { + Y_FAIL("Unknown proto type"); + } + ConvertMiniKQLTypeToYdbType( splitKeyType, *ydbValue->mutable_type()); @@ -210,25 +210,25 @@ void FillTableBoundaryImpl(TYdbProto& out, } } -void FillTableBoundary(Ydb::Table::DescribeTableResult& out, +void FillTableBoundary(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { - FillTableBoundaryImpl<Ydb::Table::DescribeTableResult>(out, in, splitKeyType); -} + FillTableBoundaryImpl<Ydb::Table::DescribeTableResult>(out, in, splitKeyType); +} -void FillTableBoundary(Ydb::Table::CreateTableRequest& out, +void FillTableBoundary(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType) { - FillTableBoundaryImpl<Ydb::Table::CreateTableRequest>(out, in, splitKeyType); -} - -template <typename TYdbProto> -void FillIndexDescriptionImpl(TYdbProto& out, + FillTableBoundaryImpl<Ydb::Table::CreateTableRequest>(out, in, splitKeyType); +} + +template <typename TYdbProto> +void FillIndexDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + for (const auto& tableIndex : in.GetTableIndexes()) { auto index = out.add_indexes(); - + index->set_name(tableIndex.GetName()); - + *index->mutable_index_columns() = { tableIndex.GetKeyColumnNames().begin(), tableIndex.GetKeyColumnNames().end() @@ -239,129 +239,129 @@ void FillIndexDescriptionImpl(TYdbProto& out, tableIndex.GetDataColumnNames().end() }; - switch (tableIndex.GetType()) { + switch (tableIndex.GetType()) { case NKikimrSchemeOp::EIndexType::EIndexTypeGlobal: *index->mutable_global_index() = Ydb::Table::GlobalIndex(); - break; + break; case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: *index->mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; - default: - break; - }; - - if constexpr (std::is_same<TYdbProto, Ydb::Table::DescribeTableResult>::value) { + break; + default: + break; + }; + + if constexpr (std::is_same<TYdbProto, Ydb::Table::DescribeTableResult>::value) { if (tableIndex.GetState() == NKikimrSchemeOp::EIndexState::EIndexStateReady) { - index->set_status(Ydb::Table::TableIndexDescription::STATUS_READY); - } else { - index->set_status(Ydb::Table::TableIndexDescription::STATUS_BUILDING); - } - index->set_size_bytes(tableIndex.GetDataSize()); + index->set_status(Ydb::Table::TableIndexDescription::STATUS_READY); + } else { + index->set_status(Ydb::Table::TableIndexDescription::STATUS_BUILDING); + } + index->set_size_bytes(tableIndex.GetDataSize()); } } } -void FillIndexDescription(Ydb::Table::DescribeTableResult& out, +void FillIndexDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillIndexDescriptionImpl(out, in); -} - -void FillIndexDescription(Ydb::Table::CreateTableRequest& out, + FillIndexDescriptionImpl(out, in); +} + +void FillIndexDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillIndexDescriptionImpl(out, in); -} - + FillIndexDescriptionImpl(out, in); +} + bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, - const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error) { - - auto returnError = [&status, &error](Ydb::StatusIds::StatusCode code, const TString& msg) -> bool { - status = code; - error = msg; - return false; - }; - + const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error) { + + auto returnError = [&status, &error](Ydb::StatusIds::StatusCode code, const TString& msg) -> bool { + status = code; + error = msg; + return false; + }; + for (const auto& index : in.indexes()) { auto indexDesc = out.MutableIndexDescription()->Add(); - - if (!index.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { + + if (!index.data_columns().empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { return returnError(Ydb::StatusIds::UNSUPPORTED, "Data column feature is not supported yet"); - } - - // common fields - indexDesc->SetName(index.name()); - - for (const auto& col : index.index_columns()) { - indexDesc->AddKeyColumnNames(col); - } - + } + + // common fields + indexDesc->SetName(index.name()); + + for (const auto& col : index.index_columns()) { + indexDesc->AddKeyColumnNames(col); + } + for (const auto& col : index.data_columns()) { indexDesc->AddDataColumnNames(col); } - // specific fields - switch (index.type_case()) { - case Ydb::Table::TableIndex::kGlobalIndex: + // specific fields + switch (index.type_case()) { + case Ydb::Table::TableIndex::kGlobalIndex: indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobal); - break; - - case Ydb::Table::TableIndex::kGlobalAsyncIndex: - if (!AppData()->FeatureFlags.GetEnableAsyncIndexes()) { - return returnError(Ydb::StatusIds::UNSUPPORTED, "Async indexes are not supported yet"); + break; + + case Ydb::Table::TableIndex::kGlobalAsyncIndex: + if (!AppData()->FeatureFlags.GetEnableAsyncIndexes()) { + return returnError(Ydb::StatusIds::UNSUPPORTED, "Async indexes are not supported yet"); } - + indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync); - break; - - default: - // pass through - // TODO: maybe return BAD_REQUEST? - break; - } - - //Disabled for a while. Probably we need to allow set this profile to user -/* - auto indexTableDesc = indexDesc->MutableIndexImplTableDescription(); - if (index.has_global_index() && index.global_index().has_table_profile()) { - auto indexTableProfile = index.global_index().table_profile(); - // Copy to common table profile to reuse common ApplyTableProfile method - Ydb::Table::TableProfile profile; - profile.mutable_partitioning_policy()->CopyFrom(indexTableProfile.partitioning_policy()); - - StatusIds::StatusCode code; - TString error; - if (!Profiles.ApplyTableProfile(profile, *indexTableDesc, code, error)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); - } - } -*/ - } - - return true; -} - -void FillTableStats(Ydb::Table::DescribeTableResult& out, + break; + + default: + // pass through + // TODO: maybe return BAD_REQUEST? + break; + } + + //Disabled for a while. Probably we need to allow set this profile to user +/* + auto indexTableDesc = indexDesc->MutableIndexImplTableDescription(); + if (index.has_global_index() && index.global_index().has_table_profile()) { + auto indexTableProfile = index.global_index().table_profile(); + // Copy to common table profile to reuse common ApplyTableProfile method + Ydb::Table::TableProfile profile; + profile.mutable_partitioning_policy()->CopyFrom(indexTableProfile.partitioning_policy()); + + StatusIds::StatusCode code; + TString error; + if (!Profiles.ApplyTableProfile(profile, *indexTableDesc, code, error)) { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } + } +*/ + } + + return true; +} + +void FillTableStats(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in, bool withPartitionStatistic) { - + auto stats = out.mutable_table_stats(); if (withPartitionStatistic) { for (const auto& tablePartitionStat : in.GetTablePartitionStats()) { auto partition = stats->add_partition_stats(); partition->set_rows_estimate(tablePartitionStat.GetRowCount()); - partition->set_store_size(tablePartitionStat.GetDataSize() + tablePartitionStat.GetIndexSize()); + partition->set_store_size(tablePartitionStat.GetDataSize() + tablePartitionStat.GetIndexSize()); } } stats->set_rows_estimate(in.GetTableStats().GetRowCount()); stats->set_partitions(in.GetTableStats().GetPartCount()); - stats->set_store_size(in.GetTableStats().GetDataSize() + in.GetTableStats().GetIndexSize()); - for (const auto& index : in.GetTable().GetTableIndexes()) { - stats->set_store_size(stats->store_size() + index.GetDataSize()); - } - + stats->set_store_size(in.GetTableStats().GetDataSize() + in.GetTableStats().GetIndexSize()); + for (const auto& index : in.GetTable().GetTableIndexes()) { + stats->set_store_size(stats->store_size() + index.GetDataSize()); + } + ui64 modificationTimeMs = in.GetTableStats().GetLastUpdateTime(); if (modificationTimeMs) { auto modificationTime = MillisecToProtoTimeStamp(modificationTimeMs); @@ -385,10 +385,10 @@ static bool IsDefaultFamily(const NKikimrSchemeOp::TFamilyDescription& family) { return false; } -template <typename TYdbProto> -void FillStorageSettingsImpl(TYdbProto& out, +template <typename TYdbProto> +void FillStorageSettingsImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + if (!in.HasPartitionConfig()) { return; } @@ -444,20 +444,20 @@ void FillStorageSettingsImpl(TYdbProto& out, } } -void FillStorageSettings(Ydb::Table::DescribeTableResult& out, +void FillStorageSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillStorageSettingsImpl(out, in); -} - -void FillStorageSettings(Ydb::Table::CreateTableRequest& out, + FillStorageSettingsImpl(out, in); +} + +void FillStorageSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillStorageSettingsImpl(out, in); -} - -template <typename TYdbProto> -void FillColumnFamiliesImpl(TYdbProto& out, + FillStorageSettingsImpl(out, in); +} + +template <typename TYdbProto> +void FillColumnFamiliesImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + if (!in.HasPartitionConfig()) { return; } @@ -510,52 +510,52 @@ void FillColumnFamiliesImpl(TYdbProto& out, } } -void FillColumnFamilies(Ydb::Table::DescribeTableResult& out, +void FillColumnFamilies(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillColumnFamiliesImpl(out, in); -} - -void FillColumnFamilies(Ydb::Table::CreateTableRequest& out, + FillColumnFamiliesImpl(out, in); +} + +void FillColumnFamilies(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillColumnFamiliesImpl(out, in); -} - -template <typename TYdbProto> -void FillAttributesImpl(TYdbProto& out, + FillColumnFamiliesImpl(out, in); +} + +template <typename TYdbProto> +void FillAttributesImpl(TYdbProto& out, const NKikimrSchemeOp::TPathDescription& in) { - - if (!in.UserAttributesSize()) { - return; - } - - auto& outAttrs = *out.mutable_attributes(); - for (const auto& inAttr : in.GetUserAttributes()) { - outAttrs[inAttr.GetKey()] = inAttr.GetValue(); - } -} - -void FillAttributes(Ydb::Table::DescribeTableResult& out, + + if (!in.UserAttributesSize()) { + return; + } + + auto& outAttrs = *out.mutable_attributes(); + for (const auto& inAttr : in.GetUserAttributes()) { + outAttrs[inAttr.GetKey()] = inAttr.GetValue(); + } +} + +void FillAttributes(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in) { - FillAttributesImpl(out, in); -} - -void FillAttributes(Ydb::Table::CreateTableRequest& out, + FillAttributesImpl(out, in); +} + +void FillAttributes(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TPathDescription& in) { - FillAttributesImpl(out, in); -} - -template <typename TYdbProto> -static void FillDefaultPartitioningSettings(TYdbProto& out) { + FillAttributesImpl(out, in); +} + +template <typename TYdbProto> +static void FillDefaultPartitioningSettings(TYdbProto& out) { // (!) We assume that all partitioning methods are disabled by default. But we don't know it for sure. auto& outPartSettings = *out.mutable_partitioning_settings(); outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); outPartSettings.set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); } -template <typename TYdbProto> -void FillPartitioningSettingsImpl(TYdbProto& out, +template <typename TYdbProto> +void FillPartitioningSettingsImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + if (!in.HasPartitionConfig()) { FillDefaultPartitioningSettings(out); return; @@ -596,39 +596,39 @@ void FillPartitioningSettingsImpl(TYdbProto& out, if (inPartPolicy.HasMaxPartitionsCount() && inPartPolicy.GetMaxPartitionsCount()) { outPartSettings.set_max_partitions_count(inPartPolicy.GetMaxPartitionsCount()); } -} +} -void FillPartitioningSettings(Ydb::Table::DescribeTableResult& out, +void FillPartitioningSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillPartitioningSettingsImpl(out, in); + FillPartitioningSettingsImpl(out, in); } -void FillPartitioningSettings(Ydb::Table::CreateTableRequest& out, +void FillPartitioningSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillPartitioningSettingsImpl(out, in); -} - + FillPartitioningSettingsImpl(out, in); +} + bool CopyExplicitPartitions(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::ExplicitPartitions& in, Ydb::StatusIds::StatusCode& status, TString& error) { - - try { - for (auto &point : in.split_points()) { - auto &dst = *out.AddSplitBoundary()->MutableKeyPrefix(); - ConvertYdbValueToMiniKQLValue(point.type(), point.value(), dst); - } - } catch (const std::exception &e) { - status = Ydb::StatusIds::BAD_REQUEST; - error = TString("cannot convert split points: ") + e.what(); - return false; - } - - return true; -} - -template <typename TYdbProto> -void FillKeyBloomFilterImpl(TYdbProto& out, + const Ydb::Table::ExplicitPartitions& in, Ydb::StatusIds::StatusCode& status, TString& error) { + + try { + for (auto &point : in.split_points()) { + auto &dst = *out.AddSplitBoundary()->MutableKeyPrefix(); + ConvertYdbValueToMiniKQLValue(point.type(), point.value(), dst); + } + } catch (const std::exception &e) { + status = Ydb::StatusIds::BAD_REQUEST; + error = TString("cannot convert split points: ") + e.what(); + return false; + } + + return true; +} + +template <typename TYdbProto> +void FillKeyBloomFilterImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + if (!in.HasPartitionConfig()) { return; } @@ -645,20 +645,20 @@ void FillKeyBloomFilterImpl(TYdbProto& out, } } -void FillKeyBloomFilter(Ydb::Table::DescribeTableResult& out, +void FillKeyBloomFilter(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillKeyBloomFilterImpl(out, in); -} - -void FillKeyBloomFilter(Ydb::Table::CreateTableRequest& out, + FillKeyBloomFilterImpl(out, in); +} + +void FillKeyBloomFilter(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillKeyBloomFilterImpl(out, in); -} - -template <typename TYdbProto> -void FillReadReplicasSettingsImpl(TYdbProto& out, + FillKeyBloomFilterImpl(out, in); +} + +template <typename TYdbProto> +void FillReadReplicasSettingsImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - + if (!in.HasPartitionConfig()) { return; } @@ -686,49 +686,49 @@ void FillReadReplicasSettingsImpl(TYdbProto& out, } } -void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, +void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { - FillReadReplicasSettingsImpl(out, in); -} + FillReadReplicasSettingsImpl(out, in); +} -void FillReadReplicasSettings(Ydb::Table::CreateTableRequest& out, +void FillReadReplicasSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { - FillReadReplicasSettingsImpl(out, in); -} - -bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, - const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error) -{ - auto& tableDesc = *out.MutableCreateTable(); - - if (!FillColumnDescription(tableDesc, in.columns(), status, error)) { - return false; - } - - tableDesc.MutableKeyColumnNames()->CopyFrom(in.primary_key()); - - TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); - if (in.has_storage_settings() && !families.ApplyStorageSettings(in.storage_settings(), &status, &error)) { - return false; - } - for (const auto& familySettings : in.column_families()) { - if (!families.ApplyFamilySettings(familySettings, &status, &error)) { - return false; - } - } - - for (auto [key, value] : in.attributes()) { - auto& attr = *out.MutableAlterUserAttributes()->AddUserAttributes(); - attr.SetKey(key); - attr.SetValue(value); - } - - TList<TString> warnings; - if (!FillCreateTableSettingsDesc(tableDesc, in, status, error, warnings, false)) { - return false; - } - - return true; -} - + FillReadReplicasSettingsImpl(out, in); +} + +bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, + const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error) +{ + auto& tableDesc = *out.MutableCreateTable(); + + if (!FillColumnDescription(tableDesc, in.columns(), status, error)) { + return false; + } + + tableDesc.MutableKeyColumnNames()->CopyFrom(in.primary_key()); + + TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); + if (in.has_storage_settings() && !families.ApplyStorageSettings(in.storage_settings(), &status, &error)) { + return false; + } + for (const auto& familySettings : in.column_families()) { + if (!families.ApplyFamilySettings(familySettings, &status, &error)) { + return false; + } + } + + for (auto [key, value] : in.attributes()) { + auto& attr = *out.MutableAlterUserAttributes()->AddUserAttributes(); + attr.SetKey(key); + attr.SetValue(value); + } + + TList<TString> warnings; + if (!FillCreateTableSettingsDesc(tableDesc, in, status, error, warnings, false)) { + return false; + } + + return true; +} + } // namespace NKikimr diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index 138f08b98fd..af77a93776a 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -1,82 +1,82 @@ -#pragma once - +#pragma once + #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> - -namespace NKikimr { - -// out -void FillColumnDescription(Ydb::Table::DescribeTableResult& out, + +namespace NKikimr { + +// out +void FillColumnDescription(Ydb::Table::DescribeTableResult& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in); -void FillColumnDescription(Ydb::Table::CreateTableRequest& out, +void FillColumnDescription(Ydb::Table::CreateTableRequest& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in); -// in +// in bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error); bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); - -// out -void FillTableBoundary(Ydb::Table::DescribeTableResult& out, + +// out +void FillTableBoundary(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType); -void FillTableBoundary(Ydb::Table::CreateTableRequest& out, +void FillTableBoundary(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, const NKikimrMiniKQL::TType& splitKeyType); - -// out -void FillIndexDescription(Ydb::Table::DescribeTableResult& out, + +// out +void FillIndexDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillIndexDescription(Ydb::Table::CreateTableRequest& out, +void FillIndexDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); -// in +// in bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, - const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error); - -// out -void FillTableStats(Ydb::Table::DescribeTableResult& out, + const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error); + +// out +void FillTableStats(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in, bool withPartitionStatistic); - -// out -void FillStorageSettings(Ydb::Table::DescribeTableResult& out, + +// out +void FillStorageSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillStorageSettings(Ydb::Table::CreateTableRequest& out, +void FillStorageSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); - -// out -void FillColumnFamilies(Ydb::Table::DescribeTableResult& out, + +// out +void FillColumnFamilies(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillColumnFamilies(Ydb::Table::CreateTableRequest& out, +void FillColumnFamilies(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); - -// out -void FillAttributes(Ydb::Table::DescribeTableResult& out, + +// out +void FillAttributes(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in); -void FillAttributes(Ydb::Table::CreateTableRequest& out, +void FillAttributes(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TPathDescription& in); - -// out -void FillPartitioningSettings(Ydb::Table::DescribeTableResult& out, + +// out +void FillPartitioningSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillPartitioningSettings(Ydb::Table::CreateTableRequest& out, +void FillPartitioningSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); - -// in + +// in bool CopyExplicitPartitions(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::ExplicitPartitions& in, Ydb::StatusIds::StatusCode& status, TString& error); - -// out -void FillKeyBloomFilter(Ydb::Table::DescribeTableResult& out, + const Ydb::Table::ExplicitPartitions& in, Ydb::StatusIds::StatusCode& status, TString& error); + +// out +void FillKeyBloomFilter(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillKeyBloomFilter(Ydb::Table::CreateTableRequest& out, +void FillKeyBloomFilter(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); - -// out -void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, + +// out +void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in); -void FillReadReplicasSettings(Ydb::Table::CreateTableRequest& out, +void FillReadReplicasSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in); - -// in -bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, - const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error); - -} // namespace NKikimr + +// in +bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, + const Ydb::Table::CreateTableRequest& in, Ydb::StatusIds::StatusCode& status, TString& error); + +} // namespace NKikimr diff --git a/ydb/core/ydb_convert/table_settings.cpp b/ydb/core/ydb_convert/table_settings.cpp index e50a7e504cc..3f3547f1f5e 100644 --- a/ydb/core/ydb_convert/table_settings.cpp +++ b/ydb/core/ydb_convert/table_settings.cpp @@ -1,7 +1,7 @@ -#include "table_description.h" +#include "table_description.h" #include "table_settings.h" -#include <util/generic/list.h> +#include <util/generic/list.h> #include <util/string/builder.h> namespace NKikimr { @@ -28,8 +28,8 @@ namespace { } bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, - const Ydb::Table::CreateTableRequest& proto, - Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings, bool tableProfileSet) + const Ydb::Table::CreateTableRequest& proto, + Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings, bool tableProfileSet) { auto &partitionConfig = *tableDesc.MutablePartitionConfig(); @@ -129,7 +129,7 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, tableDesc.SetUniformPartitionsCount(proto.uniform_partitions()); break; case Ydb::Table::CreateTableRequest::kPartitionAtKeys: - if (!CopyExplicitPartitions(tableDesc, proto.partition_at_keys(), code, error)) { + if (!CopyExplicitPartitions(tableDesc, proto.partition_at_keys(), code, error)) { return false; } break; @@ -190,18 +190,18 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, } } - if (proto.has_ttl_settings()) { - if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.ttl_settings(), code, error)) { - return false; - } - } - + if (proto.has_ttl_settings()) { + if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.ttl_settings(), code, error)) { + return false; + } + } + return true; } bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, - const Ydb::Table::AlterTableRequest& proto, - Ydb::StatusIds::StatusCode& code, TString& error, bool changed) + const Ydb::Table::AlterTableRequest& proto, + Ydb::StatusIds::StatusCode& code, TString& error, bool changed) { bool hadPartitionConfig = tableDesc.HasPartitionConfig(); auto &partitionConfig = *tableDesc.MutablePartitionConfig(); @@ -338,18 +338,18 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, changed = true; } - if (proto.has_set_ttl_settings()) { - if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.set_ttl_settings(), code, error)) { - return false; - } - } else if (proto.has_drop_ttl_settings()) { - tableDesc.MutableTTLSettings()->MutableDisabled(); - } - + if (proto.has_set_ttl_settings()) { + if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.set_ttl_settings(), code, error)) { + return false; + } + } else if (proto.has_drop_ttl_settings()) { + tableDesc.MutableTTLSettings()->MutableDisabled(); + } + if (!changed && !hadPartitionConfig) { tableDesc.ClearPartitionConfig(); } - + return true; } diff --git a/ydb/core/ydb_convert/table_settings.h b/ydb/core/ydb_convert/table_settings.h index c01d4aea0cc..cc9b4fc488a 100644 --- a/ydb/core/ydb_convert/table_settings.h +++ b/ydb/core/ydb_convert/table_settings.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> - -#include <util/datetime/base.h> + +#include <util/datetime/base.h> #include <util/string/builder.h> -namespace NKikimr { - -void MEWarning(const TString& settingName, TList<TString>& warnings); - +namespace NKikimr { + +void MEWarning(const TString& settingName, TList<TString>& warnings); + bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::CreateTableRequest& in, - Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings, bool tableProfileSet); - + const Ydb::Table::CreateTableRequest& in, + Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings, bool tableProfileSet); + bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, - const Ydb::Table::AlterTableRequest& in, - Ydb::StatusIds::StatusCode& code, TString& error, bool changed); - + const Ydb::Table::AlterTableRequest& in, + Ydb::StatusIds::StatusCode& code, TString& error, bool changed); + template <class TTtlSettingsEnabled> bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in, Ydb::StatusIds::StatusCode& code, TString& error) @@ -61,12 +61,12 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in } if constexpr (std::is_same_v<TTtlSettingsEnabled, NKikimrSchemeOp::TTTLSettings>) { - if (in.run_interval_seconds()) { - out.MutableSysSettings()->SetRunInterval(TDuration::Seconds(in.run_interval_seconds()).GetValue()); - } - } - + if (in.run_interval_seconds()) { + out.MutableSysSettings()->SetRunInterval(TDuration::Seconds(in.run_interval_seconds()).GetValue()); + } + } + return true; } -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/ydb_convert/ut/ya.make b/ydb/core/ydb_convert/ut/ya.make index cd28e5ff0f9..d71f89e815f 100644 --- a/ydb/core/ydb_convert/ut/ya.make +++ b/ydb/core/ydb_convert/ut/ya.make @@ -1,20 +1,20 @@ UNITTEST_FOR(ydb/core/ydb_convert) - + OWNER(g:kikimr) - -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - SIZE(MEDIUM) -ENDIF() - -SRCS( - ydb_convert_ut.cpp -) - -PEERDIR( - library/cpp/testing/unittest + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +SRCS( + ydb_convert_ut.cpp +) + +PEERDIR( + library/cpp/testing/unittest ydb/core/testlib -) - -END() +) + +END() diff --git a/ydb/core/ydb_convert/ya.make b/ydb/core/ydb_convert/ya.make index f65e785c6bd..230db487443 100644 --- a/ydb/core/ydb_convert/ya.make +++ b/ydb/core/ydb_convert/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - column_families.cpp - table_settings.cpp - table_description.cpp - ydb_convert.cpp -) - -PEERDIR( + +SRCS( + column_families.cpp + table_settings.cpp + table_description.cpp + ydb_convert.cpp +) + +PEERDIR( ydb/core/base ydb/core/engine ydb/core/protos @@ -20,8 +20,8 @@ PEERDIR( ydb/library/yql/minikql/dom ydb/library/yql/public/udf ydb/public/api/protos -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp index 018b638369d..11c804cff50 100644 --- a/ydb/core/ydb_convert/ydb_convert.cpp +++ b/ydb/core/ydb_convert/ydb_convert.cpp @@ -1,5 +1,5 @@ -#include "ydb_convert.h" - +#include "ydb_convert.h" + #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/base/kikimr_issue.h> @@ -782,7 +782,7 @@ TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32 mask) { void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme::Entry* to, bool processAcl) { to->set_name(from.GetName()); to->set_owner(from.GetOwner()); - + switch (from.GetPathType()) { case NKikimrSchemeOp::EPathTypeExtSubDomain: to->set_type(static_cast<Ydb::Scheme::Entry::Type>(NKikimrSchemeOp::EPathTypeSubDomain)); @@ -793,27 +793,27 @@ void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme:: if (processAcl) { const bool isDir = from.GetPathType() == NKikimrSchemeOp::EPathTypeDir; - ConvertAclToYdb(from.GetOwner(), from.GetEffectiveACL(), isDir, to->mutable_effective_permissions()); - ConvertAclToYdb(from.GetOwner(), from.GetACL(), isDir, to->mutable_permissions()); - } -} + ConvertAclToYdb(from.GetOwner(), from.GetEffectiveACL(), isDir, to->mutable_effective_permissions()); + ConvertAclToYdb(from.GetOwner(), from.GetACL(), isDir, to->mutable_permissions()); + } +} void ConvertDirectoryEntry(const NKikimrSchemeOp::TPathDescription& from, Ydb::Scheme::Entry* to, bool processAcl) { - ConvertDirectoryEntry(from.GetSelf(), to, processAcl); - - switch (from.GetSelf().GetPathType()) { + ConvertDirectoryEntry(from.GetSelf(), to, processAcl); + + switch (from.GetSelf().GetPathType()) { case NKikimrSchemeOp::EPathTypeTable: - to->set_size_bytes(from.GetTableStats().GetDataSize() + from.GetTableStats().GetIndexSize()); - for (const auto& index : from.GetTable().GetTableIndexes()) { - to->set_size_bytes(to->size_bytes() + index.GetDataSize()); - } - break; + to->set_size_bytes(from.GetTableStats().GetDataSize() + from.GetTableStats().GetIndexSize()); + for (const auto& index : from.GetTable().GetTableIndexes()) { + to->set_size_bytes(to->size_bytes() + index.GetDataSize()); + } + break; case NKikimrSchemeOp::EPathTypeSubDomain: case NKikimrSchemeOp::EPathTypeExtSubDomain: - to->set_size_bytes(from.GetDomainDescription().GetDiskSpaceUsage().GetTables().GetTotalSize()); - break; - default: - break; + to->set_size_bytes(from.GetDomainDescription().GetDiskSpaceUsage().GetTables().GetTotalSize()); + break; + default: + break; } } diff --git a/ydb/core/ydb_convert/ydb_convert.h b/ydb/core/ydb_convert/ydb_convert.h index c17de5814b1..2cf74135af7 100644 --- a/ydb/core/ydb_convert/ydb_convert.h +++ b/ydb/core/ydb_convert/ydb_convert.h @@ -39,5 +39,5 @@ TVector<TString> ConvertACLMaskToYdbPermissionNames(ui32); void ConvertDirectoryEntry(const NKikimrSchemeOp::TDirEntry& from, Ydb::Scheme::Entry* to, bool processAcl); void ConvertDirectoryEntry(const NKikimrSchemeOp::TPathDescription& from, Ydb::Scheme::Entry* to, bool processAcl); - + } // namespace NKikimr diff --git a/ydb/core/ymq/actor/action.h b/ydb/core/ymq/actor/action.h index 2dc7da09646..1801c98d68f 100644 --- a/ydb/core/ymq/actor/action.h +++ b/ydb/core/ymq/actor/action.h @@ -582,7 +582,7 @@ private: if (navigate->ErrorCount > 0) { const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; - RLOG_SQS_ERROR("Failed to read ACL for " << GetActionACLSourcePath() << ". Scheme cache error: " << status); + RLOG_SQS_ERROR("Failed to read ACL for " << GetActionACLSourcePath() << ". Scheme cache error: " << status); if (status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) { MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED); diff --git a/ydb/core/ymq/actor/create_queue.cpp b/ydb/core/ymq/actor/create_queue.cpp index 3b208a4085f..2d4b3af214e 100644 --- a/ydb/core/ymq/actor/create_queue.cpp +++ b/ydb/core/ymq/actor/create_queue.cpp @@ -41,12 +41,12 @@ private: bool DoValidate() override { auto* result = Response_.MutableCreateQueue(); - if (!IsCloud() && !UserExists_) { - MakeError(result, NErrors::OPT_IN_REQUIRED, "The specified account does not exist."); - return false; - } - - TAttribute fifo; + if (!IsCloud() && !UserExists_) { + MakeError(result, NErrors::OPT_IN_REQUIRED, "The specified account does not exist."); + return false; + } + + TAttribute fifo; for (const auto& attr : Request().attributes()) { if (attr.GetName() == "FifoQueue") { fifo = attr; diff --git a/ydb/core/ymq/actor/migration.cpp b/ydb/core/ymq/actor/migration.cpp index b5a95590dca..e547e975581 100644 --- a/ydb/core/ymq/actor/migration.cpp +++ b/ydb/core/ymq/actor/migration.cpp @@ -128,7 +128,7 @@ void TAddColumnActor::HandleTableInfo(TEvTxProxySchemeCache::TEvNavigateKeySetRe if (navigate->ErrorCount > 0) { LOG_SQS_ERROR("Queue " << TLogQueueName(UserName, QueueName) << " migration. Failed to get table \"" << TablePath << "\" info: " - << entry.Status); + << entry.Status); SendReplyAndDie(false); } else { LOG_SQS_DEBUG("Queue " << TLogQueueName(UserName, QueueName) diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp index e5dceeebf62..34c9a4ae46b 100644 --- a/ydb/core/ymq/actor/service.cpp +++ b/ydb/core/ymq/actor/service.cpp @@ -365,10 +365,10 @@ STATEFN(TSqsService::StateFunc) { } void TSqsService::InitSchemeCache() { - LOG_SQS_DEBUG("Enable scheme board scheme cache"); - auto cacheCounters = GetServiceCounters(AppData()->Counters, "sqs")->GetSubgroup("subsystem", "schemecache"); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(), cacheCounters); - SchemeCache_ = Register(CreateSchemeBoardSchemeCache(cacheConfig.Get())); + LOG_SQS_DEBUG("Enable scheme board scheme cache"); + auto cacheCounters = GetServiceCounters(AppData()->Counters, "sqs")->GetSubgroup("subsystem", "schemecache"); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(), cacheCounters); + SchemeCache_ = Register(CreateSchemeBoardSchemeCache(cacheConfig.Get())); } void TSqsService::ScheduleRequestSqsUsersList() { diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 95cfb0e0e73..676cc831ee2 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -32,7 +32,7 @@ namespace NYdb::NBackup { static constexpr const char *SCHEME_FILE_NAME = "scheme.pb"; static constexpr const char *INCOMPLETE_DATA_FILE_NAME = "incomplete.csv"; static constexpr const char *INCOMPLETE_FILE_NAME = "incomplete"; -static constexpr const char *EMPTY_FILE_NAME = "empty_dir"; +static constexpr const char *EMPTY_FILE_NAME = "empty_dir"; static constexpr size_t IO_BUFFER_SIZE = 2 << 20; // 2 MiB static constexpr i64 FILE_SPLIT_THRESHOLD = 128 << 20; // 128 MiB @@ -121,7 +121,7 @@ void PrintPrimitive(IOutputStream& out, const TValueParser& parser) { CASE_PRINT_PRIMITIVE_TYPE(out, Uint64); CASE_PRINT_PRIMITIVE_TYPE(out, Float); CASE_PRINT_PRIMITIVE_TYPE(out, Double); - CASE_PRINT_PRIMITIVE_TYPE(out, DyNumber); + CASE_PRINT_PRIMITIVE_TYPE(out, DyNumber); CASE_PRINT_PRIMITIVE_TYPE(out, Date); CASE_PRINT_PRIMITIVE_TYPE(out, Datetime); CASE_PRINT_PRIMITIVE_TYPE(out, Timestamp); @@ -133,7 +133,7 @@ void PrintPrimitive(IOutputStream& out, const TValueParser& parser) { CASE_PRINT_PRIMITIVE_STRING_TYPE(out, Utf8); CASE_PRINT_PRIMITIVE_STRING_TYPE(out, Yson); CASE_PRINT_PRIMITIVE_STRING_TYPE(out, Json); - CASE_PRINT_PRIMITIVE_STRING_TYPE(out, JsonDocument); + CASE_PRINT_PRIMITIVE_STRING_TYPE(out, JsonDocument); default: Y_FAIL("Unsupported type"); } @@ -360,8 +360,8 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable NTable::TTableClient client(driver); TStatus status = client.RetryOperationSync([fullTablePath, &desc](NTable::TSession session) { - auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true); - auto result = session.DescribeTable(fullTablePath, settings).GetValueSync(); + auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true); + auto result = session.DescribeTable(fullTablePath, settings).GetValueSync(); VerifyStatus(result); desc = result.GetTableDescription(); @@ -376,44 +376,44 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable return *desc; } -Ydb::Table::CreateTableRequest ProtoFromTableDescription(const NTable::TTableDescription& desc, bool preservePoolKinds) { +Ydb::Table::CreateTableRequest ProtoFromTableDescription(const NTable::TTableDescription& desc, bool preservePoolKinds) { Ydb::Table::CreateTableRequest proto; - desc.SerializeTo(proto); - - if (preservePoolKinds) { - return proto; - } - - if (proto.has_profile()) { - auto& profile = *proto.mutable_profile(); - - if (profile.has_storage_policy()) { - auto& policy = *profile.mutable_storage_policy(); - - policy.clear_syslog(); - policy.clear_log(); - policy.clear_data(); - policy.clear_external(); - - for (auto& family : *policy.mutable_column_families()) { - family.clear_data(); - family.clear_external(); - } - } - } - - if (proto.has_storage_settings()) { - auto& settings = *proto.mutable_storage_settings(); - - settings.clear_tablet_commit_log0(); - settings.clear_tablet_commit_log1(); - settings.clear_external(); - } - - for (auto& family : *proto.mutable_column_families()) { - family.clear_data(); - } - + desc.SerializeTo(proto); + + if (preservePoolKinds) { + return proto; + } + + if (proto.has_profile()) { + auto& profile = *proto.mutable_profile(); + + if (profile.has_storage_policy()) { + auto& policy = *profile.mutable_storage_policy(); + + policy.clear_syslog(); + policy.clear_log(); + policy.clear_data(); + policy.clear_external(); + + for (auto& family : *policy.mutable_column_families()) { + family.clear_data(); + family.clear_external(); + } + } + } + + if (proto.has_storage_settings()) { + auto& settings = *proto.mutable_storage_settings(); + + settings.clear_tablet_commit_log0(); + settings.clear_tablet_commit_log1(); + settings.clear_external(); + } + + for (auto& family : *proto.mutable_column_families()) { + family.clear_data(); + } + return proto; } @@ -469,7 +469,7 @@ void DropTable(TDriver driver, const TString& path) { } void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupPrefix, const TString& path, - const TFsPath& folderPath, bool schemaOnly, bool preservePoolKinds) { + const TFsPath& folderPath, bool schemaOnly, bool preservePoolKinds) { Y_ENSURE(!path.empty()); Y_ENSURE(path.back() != '/', path.Quote() << " path contains / in the end"); @@ -477,7 +477,7 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP << " backupPrefix: " << backupPrefix << " path: " << path); auto desc = DescribeTable(driver, JoinDatabasePath(schemaOnly ? dbPrefix : backupPrefix, path)); - auto proto = ProtoFromTableDescription(desc, preservePoolKinds); + auto proto = ProtoFromTableDescription(desc, preservePoolKinds); TString schemaStr; google::protobuf::TextFormat::PrintToString(proto, &schemaStr); @@ -513,19 +513,19 @@ void RemoveClusterDirectoryRecursive(const TDriver& driver, const TString& path) LOG_DEBUG("Directory is removed recursively, path: " << path.Quote()); } -static bool IsExcluded(const TString& path, const TVector<TRegExMatch>& exclusionPatterns) { - for (const auto& pattern : exclusionPatterns) { - if (pattern.Match(path.c_str())) { - return true; - } - } - - return false; -} - +static bool IsExcluded(const TString& path, const TVector<TRegExMatch>& exclusionPatterns) { + for (const auto& pattern : exclusionPatterns) { + if (pattern.Match(path.c_str())) { + return true; + } + } + + return false; +} + void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& backupPrefix, TString path, - const TFsPath folderPath, const TVector<TRegExMatch>& exclusionPatterns, - bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool preservePoolKinds) { + const TFsPath folderPath, const TVector<TRegExMatch>& exclusionPatterns, + bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool preservePoolKinds) { LOG_DEBUG("Going to backup folder/table, dbPrefix: " << dbPrefix << " path: " << path); TFile(folderPath.Child(INCOMPLETE_FILE_NAME), CreateAlways); @@ -535,12 +535,12 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba { TDbIterator<ETraverseType::Preordering> dbIt(driver, dbPrefix); while (dbIt) { - if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { - LOG_DEBUG("skip path# " << dbIt.GetFullPath()); - dbIt.Next(); - continue; - } - + if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { + LOG_DEBUG("skip path# " << dbIt.GetFullPath()); + dbIt.Next(); + continue; + } + TFsPath childFolderPath = folderPath.Child(dbIt.GetRelPath()); LOG_DEBUG("path to backup# " << childFolderPath.GetPath()); childFolderPath.MkDir(); @@ -548,10 +548,10 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba if (schemaOnly) { if (dbIt.IsTable()) { BackupTable(driver, dbIt.GetTraverseRoot(), backupPrefix, dbIt.GetRelPath(), - childFolderPath, schemaOnly, preservePoolKinds); + childFolderPath, schemaOnly, preservePoolKinds); childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); } - } else if (!avoidCopy) { + } else if (!avoidCopy) { if (dbIt.IsTable()) { const TString tmpTablePath = JoinDatabasePath(backupPrefix, dbIt.GetRelPath()); if (useConsistentCopyTable) { @@ -560,7 +560,7 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba auto status = CopyTableAsyncStart(driver, dbIt.GetFullPath(), tmpTablePath); copiedTablesStatuses.emplace(dbIt.GetFullPath(), std::move(status)); } - } else if (dbIt.IsDir()) { + } else if (dbIt.IsDir()) { CreateClusterDirectory(driver, JoinDatabasePath(backupPrefix, dbIt.GetRelPath())); } } @@ -571,75 +571,75 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba if (schemaOnly) { TDbIterator<ETraverseType::Postordering> dbIt(driver, dbPrefix); while (dbIt) { - if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { - dbIt.Next(); - continue; - } - + if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { + dbIt.Next(); + continue; + } + TFsPath childFolderPath = folderPath.Child(dbIt.GetRelPath()); if (dbIt.IsTable()) { // If table backup was not successful exception should be thrown, // so control flow can't reach this line. Check it just to be sure Y_ENSURE(!childFolderPath.Child(INCOMPLETE_FILE_NAME).Exists()); - } else if (dbIt.IsDir()) { + } else if (dbIt.IsDir()) { childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); - - TVector<TString> children; - childFolderPath.ListNames(children); - if (children.empty()) { - TFile(childFolderPath.Child(EMPTY_FILE_NAME), CreateAlways); - } + + TVector<TString> children; + childFolderPath.ListNames(children); + if (children.empty()) { + TFile(childFolderPath.Child(EMPTY_FILE_NAME), CreateAlways); + } } - - childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); + + childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); dbIt.Next(); } folderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); return; } - if (useConsistentCopyTable && !avoidCopy && tablesToCopy) { + if (useConsistentCopyTable && !avoidCopy && tablesToCopy) { CopyTables(driver, tablesToCopy); } // Read all tables from temporal folder and delete them { TDbIterator<ETraverseType::Postordering> dbIt(driver, dbPrefix); while (dbIt) { - if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { - dbIt.Next(); - continue; - } - + if (IsExcluded(dbIt.GetFullPath(), exclusionPatterns)) { + dbIt.Next(); + continue; + } + TFsPath childFolderPath = folderPath.Child(dbIt.GetRelPath()); const TString tmpTablePath = JoinDatabasePath(backupPrefix, dbIt.GetRelPath()); - + if (dbIt.IsTable()) { - if (!useConsistentCopyTable && !avoidCopy) { + if (!useConsistentCopyTable && !avoidCopy) { // CopyTableAsyncFinish(const TAsyncStatus& status, const TString& src, const TString& dst); Y_ENSURE(copiedTablesStatuses.contains(dbIt.GetFullPath()), "Table was not copied but going to be backuped, path# " << dbIt.GetFullPath().Quote()); CopyTableAsyncFinish(copiedTablesStatuses[dbIt.GetFullPath()], dbIt.GetFullPath(), tmpTablePath); copiedTablesStatuses.erase(dbIt.GetFullPath()); } - BackupTable(driver, dbIt.GetTraverseRoot(), avoidCopy ? dbIt.GetTraverseRoot() : backupPrefix, dbIt.GetRelPath(), - childFolderPath, schemaOnly, preservePoolKinds); - if (!avoidCopy) { - DropTable(driver, tmpTablePath); - } - } else if (dbIt.IsDir()) { - childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); - - TVector<TString> children; - childFolderPath.ListNames(children); - if (children.empty()) { - TFile(childFolderPath.Child(EMPTY_FILE_NAME), CreateAlways); - } - - if (!avoidCopy) { - RemoveClusterDirectory(driver, tmpTablePath); - } + BackupTable(driver, dbIt.GetTraverseRoot(), avoidCopy ? dbIt.GetTraverseRoot() : backupPrefix, dbIt.GetRelPath(), + childFolderPath, schemaOnly, preservePoolKinds); + if (!avoidCopy) { + DropTable(driver, tmpTablePath); + } + } else if (dbIt.IsDir()) { + childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); + + TVector<TString> children; + childFolderPath.ListNames(children); + if (children.empty()) { + TFile(childFolderPath.Child(EMPTY_FILE_NAME), CreateAlways); + } + + if (!avoidCopy) { + RemoveClusterDirectory(driver, tmpTablePath); + } } - + childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists(); dbIt.Next(); } @@ -665,8 +665,8 @@ void CheckedCreateBackupFolder(const TFsPath& folderPath) { // relDbPath - relative path to directory/table to be backuped // folderPath - relative path to folder in local filesystem where backup will be stored void BackupFolder(TDriver driver, const TString& database, const TString& relDbPath, TFsPath folderPath, - const TVector<TRegExMatch>& exclusionPatterns, - bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool savePartialResult, bool preservePoolKinds) { + const TVector<TRegExMatch>& exclusionPatterns, + bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool savePartialResult, bool preservePoolKinds) { TString temporalBackupPostfix = CreateTemporalBackupName(); if (!folderPath) { folderPath = temporalBackupPostfix; @@ -684,8 +684,8 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP TString dbPrefix = JoinDatabasePath(database, relDbPath); TString path; - BackupFolderImpl(driver, dbPrefix, tmpDbFolder, path, folderPath, exclusionPatterns, - schemaOnly, useConsistentCopyTable, avoidCopy, preservePoolKinds); + BackupFolderImpl(driver, dbPrefix, tmpDbFolder, path, folderPath, exclusionPatterns, + schemaOnly, useConsistentCopyTable, avoidCopy, preservePoolKinds); } catch (...) { if (!schemaOnly && !avoidCopy) { RemoveClusterDirectoryRecursive(driver, tmpDbFolder); @@ -697,7 +697,7 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP } throw; } - if (!schemaOnly && !avoidCopy) { + if (!schemaOnly && !avoidCopy) { RemoveClusterDirectoryRecursive(driver, tmpDbFolder); } } @@ -818,8 +818,8 @@ void UploadDataIntoTable(TDriver driver, const NTable::TTableDescription& tableD TQueryFromFileIterator it(relPath, dataFileName, tableDesc.GetColumns(), IO_BUFFER_SIZE, params.MaxRowsPerQuery, params.MaxBytesPerQuery); - NTable::TTableClient client(driver); - TUploader uploader(opts, client, it.GetQueryString()); + NTable::TTableClient client(driver); + TUploader uploader(opts, client, it.GetQueryString()); if (!params.UseBulkUpsert) { LOG_DEBUG("Query string:\n" << it.GetQueryString()); } diff --git a/ydb/library/backup/backup.h b/ydb/library/backup/backup.h index 027ab82899b..269f96d9b9d 100644 --- a/ydb/library/backup/backup.h +++ b/ydb/library/backup/backup.h @@ -4,8 +4,8 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> -#include <library/cpp/regex/pcre/regexp.h> - +#include <library/cpp/regex/pcre/regexp.h> + #include <util/folder/path.h> #include <util/generic/singleton.h> #include <util/stream/output.h> @@ -33,9 +33,9 @@ public: }; void BackupFolder(TDriver driver, const TString& database, const TString& relDbPath, TFsPath folderPath, - const TVector<TRegExMatch>& exclusionPatterns, - bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy = false, bool savePartialResult = false, - bool preservePoolKinds = false); + const TVector<TRegExMatch>& exclusionPatterns, + bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy = false, bool savePartialResult = false, + bool preservePoolKinds = false); struct TRestoreFolderParams { bool OnlyCheck = false; diff --git a/ydb/library/backup/db_iterator.h b/ydb/library/backup/db_iterator.h index 0b4f8a3dd09..3abc8ecb99a 100644 --- a/ydb/library/backup/db_iterator.h +++ b/ydb/library/backup/db_iterator.h @@ -67,7 +67,7 @@ public: break; } case ETraverseType::Postordering: { - while (NextNodes && IsDir() && !IsListed()) { + while (NextNodes && IsDir() && !IsListed()) { const TString& fullPath = GetFullPath(); NScheme::TListDirectoryResult childList = Client.ListDirectory(fullPath).GetValueSync(); Y_ENSURE(childList.IsSuccess(), "Can't list directory, maybe it doesn't exist, dbPath# " diff --git a/ydb/library/backup/query_builder.cpp b/ydb/library/backup/query_builder.cpp index 8cf67d1efc0..dd931c82bbe 100644 --- a/ydb/library/backup/query_builder.cpp +++ b/ydb/library/backup/query_builder.cpp @@ -180,12 +180,12 @@ void TQueryBuilder::AddPrimitiveMember(EPrimitiveType type, TStringBuf buf) { break; case EPrimitiveType::DyNumber: - if (buf == "null") { - Value.OptionalDyNumber(Nothing()); - } else { - Y_ENSURE(NKikimr::NDyNumber::IsValidDyNumberString(buf)); - Value.OptionalDyNumber(TString(buf)); - } + if (buf == "null") { + Value.OptionalDyNumber(Nothing()); + } else { + Y_ENSURE(NKikimr::NDyNumber::IsValidDyNumberString(buf)); + Value.OptionalDyNumber(TString(buf)); + } break; case EPrimitiveType::Uuid: diff --git a/ydb/library/backup/query_uploader.cpp b/ydb/library/backup/query_uploader.cpp index ad83927b6cc..c9689c3b2e4 100644 --- a/ydb/library/backup/query_uploader.cpp +++ b/ydb/library/backup/query_uploader.cpp @@ -5,8 +5,8 @@ namespace NYdb::NBackup { -static const char DOC_API_REQUEST_TYPE[] = "_document_api_request"; - +static const char DOC_API_REQUEST_TYPE[] = "_document_api_request"; + //////////////////////////////////////////////////////////////////////////////// // TUploader //////////////////////////////////////////////////////////////////////////////// @@ -15,12 +15,12 @@ ui32 TUploader::TOptions::GetRps() const { return Rate * TDuration::Seconds(1).MilliSeconds() / Interval.MilliSeconds(); } -TUploader::TUploader(const TUploader::TOptions &opts, NYdb::NTable::TTableClient& client, const TString &query) +TUploader::TUploader(const TUploader::TOptions &opts, NYdb::NTable::TTableClient& client, const TString &query) : Opts(opts) , Query(query) , ShouldStop(0) , RequestLimiter(opts.GetRps(), opts.GetRps()) - , Client(client) + , Client(client) { TasksQueue = MakeSimpleShared<TThreadPool>(TThreadPool::TParams().SetBlocking(true).SetCatching(true)); TasksQueue->Start(opts.InFly, opts.InFly + 1); @@ -49,7 +49,7 @@ bool TUploader::Push(const TString& path, TValue&& value) { auto upsert = [&] (NYdb::NTable::TSession) -> TStatus { auto settings = NTable::TBulkUpsertSettings() - .RequestType(DOC_API_REQUEST_TYPE) + .RequestType(DOC_API_REQUEST_TYPE) .OperationTimeout(TDuration::Seconds(30)) .ClientTimeout(TDuration::Seconds(35)); @@ -102,20 +102,20 @@ bool TUploader::Push(TParams params) { } auto upload = [this, params] (NYdb::NTable::TSession session) -> NYdb::TStatus { - auto prepareSettings = NTable::TPrepareDataQuerySettings() - .RequestType(DOC_API_REQUEST_TYPE); - auto prepareResult = session.PrepareDataQuery(Query, prepareSettings).GetValueSync(); + auto prepareSettings = NTable::TPrepareDataQuerySettings() + .RequestType(DOC_API_REQUEST_TYPE); + auto prepareResult = session.PrepareDataQuery(Query, prepareSettings).GetValueSync(); if (!prepareResult.IsSuccess()) { return prepareResult; } auto dataQuery = prepareResult.GetQuery(); auto transaction = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); - auto settings = NTable::TExecDataQuerySettings() - .RequestType(DOC_API_REQUEST_TYPE) - .OperationTimeout(TDuration::Seconds(30)) - .ClientTimeout(TDuration::Seconds(35)); - return dataQuery.Execute(transaction, std::move(params), settings).GetValueSync(); + auto settings = NTable::TExecDataQuerySettings() + .RequestType(DOC_API_REQUEST_TYPE) + .OperationTimeout(TDuration::Seconds(30)) + .ClientTimeout(TDuration::Seconds(35)); + return dataQuery.Execute(transaction, std::move(params), settings).GetValueSync(); }; auto task = [this, upload] () { diff --git a/ydb/library/backup/query_uploader.h b/ydb/library/backup/query_uploader.h index 4661dec0532..b72a31cea26 100644 --- a/ydb/library/backup/query_uploader.h +++ b/ydb/library/backup/query_uploader.h @@ -34,10 +34,10 @@ private: using TRpsLimiter = TBucketQuoter<ui64>; TRpsLimiter RequestLimiter; - NYdb::NTable::TTableClient& Client; + NYdb::NTable::TTableClient& Client; public: - TUploader(const TOptions& opts, NYdb::NTable::TTableClient& client, const TString& query); + TUploader(const TOptions& opts, NYdb::NTable::TTableClient& client, const TString& query); bool Push(TParams params); bool Push(const TString& path, TValue&& value); diff --git a/ydb/library/backup/ut/ut.cpp b/ydb/library/backup/ut/ut.cpp index 23f5b76a0d7..5967e9dfa34 100644 --- a/ydb/library/backup/ut/ut.cpp +++ b/ydb/library/backup/ut/ut.cpp @@ -485,66 +485,66 @@ Y_UNIT_TEST(ResultSetDecimalPrintTest) { TestResultSetParsedOk(resultSetStr, expect); } -Y_UNIT_TEST(ResultSetDyNumberPrintTest) { - TString resultSetStr = R"_( - columns:{ - name: "Col1" - type: { type_id: DYNUMBER } - } - rows:{ items:{ text_value: "0" } } - rows:{ items:{ text_value: ".0" } } - rows:{ items:{ text_value: "0.93" } } - rows:{ items:{ text_value: "1" } } - rows:{ items:{ text_value: "-1" } } - rows:{ items:{ text_value: "724.1" } } - rows:{ items:{ text_value: "1E-130" } } - rows:{ items:{ text_value: "9.9999999999999999999999999999999999999E+125" } } - rows:{ items:{ text_value: "-1E-130" } } - rows:{ items:{ text_value: "-9.9999999999999999999999999999999999999E+125" } } - truncated: false - )_"; - - const TString expect = TStringBuilder() - << "0" << Endl - << ".0" << Endl - << "0.93" << Endl - << "1" << Endl - << "-1" << Endl - << "724.1" << Endl - << "1E-130" << Endl - << "9.9999999999999999999999999999999999999E+125" << Endl - << "-1E-130" << Endl - << "-9.9999999999999999999999999999999999999E+125" << Endl; - TestResultSetParsedOk(resultSetStr, expect); +Y_UNIT_TEST(ResultSetDyNumberPrintTest) { + TString resultSetStr = R"_( + columns:{ + name: "Col1" + type: { type_id: DYNUMBER } + } + rows:{ items:{ text_value: "0" } } + rows:{ items:{ text_value: ".0" } } + rows:{ items:{ text_value: "0.93" } } + rows:{ items:{ text_value: "1" } } + rows:{ items:{ text_value: "-1" } } + rows:{ items:{ text_value: "724.1" } } + rows:{ items:{ text_value: "1E-130" } } + rows:{ items:{ text_value: "9.9999999999999999999999999999999999999E+125" } } + rows:{ items:{ text_value: "-1E-130" } } + rows:{ items:{ text_value: "-9.9999999999999999999999999999999999999E+125" } } + truncated: false + )_"; + + const TString expect = TStringBuilder() + << "0" << Endl + << ".0" << Endl + << "0.93" << Endl + << "1" << Endl + << "-1" << Endl + << "724.1" << Endl + << "1E-130" << Endl + << "9.9999999999999999999999999999999999999E+125" << Endl + << "-1E-130" << Endl + << "-9.9999999999999999999999999999999999999E+125" << Endl; + TestResultSetParsedOk(resultSetStr, expect); +} + +Y_UNIT_TEST(ResultSetJsonDocumentPrintTest) { + TString resultSetStr = R"_( + columns:{ + name: "Col1" + type: { type_id: JSON_DOCUMENT } + } + rows:{ items:{ text_value: '{"key": "value"}' } } + rows:{ items:{ text_value: '{"key": 0}' } } + rows:{ items:{ text_value: '{"key": 0.12}' } } + rows:{ items:{ text_value: '{"key": -1}' } } + rows:{ items:{ text_value: '{"key": [1, 2]}' } } + rows:{ items:{ text_value: '{"key": {"1": "one", "2": 2}}' } } + truncated: false + )_"; + + const TString expect = TStringBuilder() + << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"" << Endl + << "\"" << CGIEscapeRet(R"({"key": 0})") << "\"" << Endl + << "\"" << CGIEscapeRet(R"({"key": 0.12})") << "\"" << Endl + << "\"" << CGIEscapeRet(R"({"key": -1})") << "\"" << Endl + << "\"" << CGIEscapeRet(R"({"key": [1, 2]})") << "\"" << Endl + << "\"" << CGIEscapeRet(R"({"key": {"1": "one", "2": 2}})") << "\"" << Endl; + TestResultSetParsedOk(resultSetStr, expect); +} + } -Y_UNIT_TEST(ResultSetJsonDocumentPrintTest) { - TString resultSetStr = R"_( - columns:{ - name: "Col1" - type: { type_id: JSON_DOCUMENT } - } - rows:{ items:{ text_value: '{"key": "value"}' } } - rows:{ items:{ text_value: '{"key": 0}' } } - rows:{ items:{ text_value: '{"key": 0.12}' } } - rows:{ items:{ text_value: '{"key": -1}' } } - rows:{ items:{ text_value: '{"key": [1, 2]}' } } - rows:{ items:{ text_value: '{"key": {"1": "one", "2": 2}}' } } - truncated: false - )_"; - - const TString expect = TStringBuilder() - << "\"" << CGIEscapeRet(R"({"key": "value"})") << "\"" << Endl - << "\"" << CGIEscapeRet(R"({"key": 0})") << "\"" << Endl - << "\"" << CGIEscapeRet(R"({"key": 0.12})") << "\"" << Endl - << "\"" << CGIEscapeRet(R"({"key": -1})") << "\"" << Endl - << "\"" << CGIEscapeRet(R"({"key": [1, 2]})") << "\"" << Endl - << "\"" << CGIEscapeRet(R"({"key": {"1": "one", "2": 2}})") << "\"" << Endl; - TestResultSetParsedOk(resultSetStr, expect); -} - -} - Y_UNIT_TEST_SUITE(UtilTest) { Y_UNIT_TEST(SizeFromStringParsing) { diff --git a/ydb/library/backup/util.cpp b/ydb/library/backup/util.cpp index db9af2ed07e..332d29ac8ee 100644 --- a/ydb/library/backup/util.cpp +++ b/ydb/library/backup/util.cpp @@ -52,20 +52,20 @@ TMap<TStringBuf, ui64> SizeSuffix { {"G", Power<ui64>(1000, 3)}, {"T", Power<ui64>(1000, 4)}, {"Ki", Power<ui64>(1024, 1)}, - {"KiB", Power<ui64>(1024, 1)}, + {"KiB", Power<ui64>(1024, 1)}, {"Mi", Power<ui64>(1024, 2)}, - {"MiB", Power<ui64>(1024, 2)}, + {"MiB", Power<ui64>(1024, 2)}, {"Gi", Power<ui64>(1024, 3)}, - {"GiB", Power<ui64>(1024, 3)}, + {"GiB", Power<ui64>(1024, 3)}, {"Ti", Power<ui64>(1024, 4)}, - {"TiB", Power<ui64>(1024, 4)}, + {"TiB", Power<ui64>(1024, 4)}, }; } ui64 SizeFromString(TStringBuf s) { size_t pos = s.Size(); - while (pos > 0 && !isdigit(s[pos - 1])) { + while (pos > 0 && !isdigit(s[pos - 1])) { --pos; } diff --git a/ydb/library/backup/ya.make b/ydb/library/backup/ya.make index ca85aa255e2..9ea7b93961a 100644 --- a/ydb/library/backup/ya.make +++ b/ydb/library/backup/ya.make @@ -7,7 +7,7 @@ OWNER( PEERDIR( library/cpp/bucket_quoter - library/cpp/regex/pcre + library/cpp/regex/pcre library/cpp/string_utils/quote util ydb/library/dynumber diff --git a/ydb/library/dynumber/cast.h b/ydb/library/dynumber/cast.h index b3b6516296a..2625ec446fc 100644 --- a/ydb/library/dynumber/cast.h +++ b/ydb/library/dynumber/cast.h @@ -1,114 +1,114 @@ -#pragma once - -#include "dynumber.h" - -#include <util/generic/ymath.h> - -#include <limits> - -namespace NKikimr::NDyNumber { - -template <typename T> -TMaybe<T> TryFromDyNumber(TStringBuf buffer) { - using traits = std::numeric_limits<T>; - - if (!traits::is_specialized || !IsValidDyNumber(buffer)) { - return Nothing(); - } - - auto result = T(); - - auto s = buffer.data(); - if (*s == '\x01') { - return result; - } - - const bool negative = !*s++; - if (negative && !traits::is_signed) { - return Nothing(); - } - - auto power = ui8(*s++); - if (negative) { - power = '\xFF' - power; - } - - std::function<TMaybe<T>(T, i16)> pow = [&pow](T base, i16 exp) -> TMaybe<T> { - if (exp > 0) { - const auto p = Power(base, exp - 1); - if (!p || p > traits::max()) { - return Nothing(); - } - - return base * p; - } else if (exp < 0) { - if (const auto p = pow(base, abs(exp))) { - return 1 / *p; - } - - return Nothing(); - } else { - return T(1); - } - }; - - auto apply = [&negative, &result, &pow](ui8 digit, i16 exp) -> bool { - if (traits::is_integer && (exp < 0 || exp > traits::digits10)) { - return false; - } - - if (digit == '\x00') { - return true; - } - - if (const auto p = pow(10, exp)) { - const T mul = digit * *p; - - if (!mul || mul < *p || mul > traits::max()) { - return false; - } - - if (negative) { - if (traits::lowest() - result > -mul) { - return false; - } - - result -= mul; - } else { - if (traits::max() - result < mul) { - return false; - } - - result += mul; - } - - return true; - } - - return false; - }; - - const auto digits = negative ? "FEDCBA9876543210" : "0123456789ABCDEF"; - - auto e = power - 129; - auto l = buffer.size() - 1; - while (--l) { - const auto c = *s++; - const auto hi = (c >> '\x04') & '\x0F'; - const auto lo = c & '\x0F'; - - if (!apply(digits[hi] - '0', --e)) { - return Nothing(); - } - - if (lo != (negative ? '\x0F' : '\x00') || l > 1U) { - if (!apply(digits[lo] - '0', --e)) { - return Nothing(); - } - } - } - - return result; -} - -} +#pragma once + +#include "dynumber.h" + +#include <util/generic/ymath.h> + +#include <limits> + +namespace NKikimr::NDyNumber { + +template <typename T> +TMaybe<T> TryFromDyNumber(TStringBuf buffer) { + using traits = std::numeric_limits<T>; + + if (!traits::is_specialized || !IsValidDyNumber(buffer)) { + return Nothing(); + } + + auto result = T(); + + auto s = buffer.data(); + if (*s == '\x01') { + return result; + } + + const bool negative = !*s++; + if (negative && !traits::is_signed) { + return Nothing(); + } + + auto power = ui8(*s++); + if (negative) { + power = '\xFF' - power; + } + + std::function<TMaybe<T>(T, i16)> pow = [&pow](T base, i16 exp) -> TMaybe<T> { + if (exp > 0) { + const auto p = Power(base, exp - 1); + if (!p || p > traits::max()) { + return Nothing(); + } + + return base * p; + } else if (exp < 0) { + if (const auto p = pow(base, abs(exp))) { + return 1 / *p; + } + + return Nothing(); + } else { + return T(1); + } + }; + + auto apply = [&negative, &result, &pow](ui8 digit, i16 exp) -> bool { + if (traits::is_integer && (exp < 0 || exp > traits::digits10)) { + return false; + } + + if (digit == '\x00') { + return true; + } + + if (const auto p = pow(10, exp)) { + const T mul = digit * *p; + + if (!mul || mul < *p || mul > traits::max()) { + return false; + } + + if (negative) { + if (traits::lowest() - result > -mul) { + return false; + } + + result -= mul; + } else { + if (traits::max() - result < mul) { + return false; + } + + result += mul; + } + + return true; + } + + return false; + }; + + const auto digits = negative ? "FEDCBA9876543210" : "0123456789ABCDEF"; + + auto e = power - 129; + auto l = buffer.size() - 1; + while (--l) { + const auto c = *s++; + const auto hi = (c >> '\x04') & '\x0F'; + const auto lo = c & '\x0F'; + + if (!apply(digits[hi] - '0', --e)) { + return Nothing(); + } + + if (lo != (negative ? '\x0F' : '\x00') || l > 1U) { + if (!apply(digits[lo] - '0', --e)) { + return Nothing(); + } + } + } + + return result; +} + +} diff --git a/ydb/library/dynumber/ut/dynumber_ut.cpp b/ydb/library/dynumber/ut/dynumber_ut.cpp index 8e04d7b7dda..f46db222de2 100644 --- a/ydb/library/dynumber/ut/dynumber_ut.cpp +++ b/ydb/library/dynumber/ut/dynumber_ut.cpp @@ -26,25 +26,25 @@ namespace { UNIT_ASSERT_EQUAL(*dyNumber, *dyNumberAfterString); } - - template <typename T> - void TestCast(TStringBuf test, TMaybe<T> value) { - UNIT_ASSERT_C(IsValidDyNumberString(test), test); - - const auto dyNumber = ParseDyNumberString(test); - UNIT_ASSERT(dyNumber.Defined()); - UNIT_ASSERT(IsValidDyNumber(*dyNumber)); - - const auto casted = TryFromDyNumber<T>(*dyNumber); - - if constexpr (std::is_integral<T>::value) { - UNIT_ASSERT_VALUES_EQUAL(casted, value); - } else if (casted && value) { - UNIT_ASSERT_DOUBLES_EQUAL(*casted, *value, 1e-9); - } else { - UNIT_ASSERT_C(!casted && !value, "Casted: " << casted << ", value: " << value); - } - } + + template <typename T> + void TestCast(TStringBuf test, TMaybe<T> value) { + UNIT_ASSERT_C(IsValidDyNumberString(test), test); + + const auto dyNumber = ParseDyNumberString(test); + UNIT_ASSERT(dyNumber.Defined()); + UNIT_ASSERT(IsValidDyNumber(*dyNumber)); + + const auto casted = TryFromDyNumber<T>(*dyNumber); + + if constexpr (std::is_integral<T>::value) { + UNIT_ASSERT_VALUES_EQUAL(casted, value); + } else if (casted && value) { + UNIT_ASSERT_DOUBLES_EQUAL(*casted, *value, 1e-9); + } else { + UNIT_ASSERT_C(!casted && !value, "Casted: " << casted << ", value: " << value); + } + } } Y_UNIT_TEST_SUITE(TDyNumberTests) { @@ -69,111 +69,111 @@ Y_UNIT_TEST_SUITE(TDyNumberTests) { TestDyNumber("-9.9999999999999999999999999999999999999E+125"); TestDyNumber("-9.9999999999999999999999999999999999999000E+125"); } - - Y_UNIT_TEST(Cast) { - TestCast<int>("0", 0); - - TestCast<int>("1", 1); - TestCast<int>("-1", -1); - - TestCast<int>("12", 12); - TestCast<int>("-12", -12); - TestCast<int>("123", 123); - TestCast<int>("-123", -123); - TestCast<int>("1234", 1234); - TestCast<int>("-1234", -1234); - - TestCast<int>(ToString(Max<int>()), Max<int>()); - TestCast<int>(ToString(Min<int>()), Min<int>()); - - TestCast<i8> ("200", Nothing()); - TestCast<i16>("40000", Nothing()); - TestCast<i32>("3000000000", Nothing()); - - TestCast<ui8> ("300", Nothing()); - TestCast<ui16>("70000", Nothing()); - TestCast<ui32>("5000000000", Nothing()); - - // int to floating point - TestCast<double>("1", 1); - TestCast<double>("12", 12); - TestCast<double>("123", 123); - - // floating point to int - TestCast<int>("0.1", Nothing()); - TestCast<int>("0.23", Nothing()); - TestCast<int>("1.2", Nothing()); - TestCast<int>("1.23", Nothing()); - TestCast<int>("12.3", Nothing()); - TestCast<int>("123.4", Nothing()); - - // double - TestCast<double>("0.1", 0.1); - TestCast<double>("0.23", 0.23); - TestCast<double>("-1.23", -1.23); - TestCast<double>("12.3", 12.3); - TestCast<double>("123.4", 123.4); - TestCast<double>("1.23E20", 1.23E20); - TestCast<double>("1.23E-20", 1.23E-20); - - // float - TestCast<float>("-0.1", -0.1f); - TestCast<float>("-0.23", -0.23f); - TestCast<float>("1.23", 1.23f); - TestCast<float>("-12.3", -12.3f); - TestCast<float>("-123.4", -123.4f); - TestCast<float>("-1.23E10", -1.23E10f); - TestCast<float>("-1.23E-10", -1.23E-10f); - - // unsigned Max - TestCast<ui8> (ToString(Max<ui8>()), Max<ui8>()); - TestCast<ui16>(ToString(Max<ui16>()), Max<ui16>()); - TestCast<ui32>(ToString(Max<ui32>()), Max<ui32>()); - TestCast<ui64>(ToString(Max<ui64>()), Max<ui64>()); - - // signed Max - TestCast<i8> (ToString(Max<i8>()), Max<i8>()); - TestCast<i16>(ToString(Max<i16>()), Max<i16>()); - TestCast<i32>(ToString(Max<i32>()), Max<i32>()); - TestCast<i64>(ToString(Max<i64>()), Max<i64>()); - - // signed Min - TestCast<i8> (ToString(Min<i8>()), Min<i8>()); - TestCast<i16>(ToString(Min<i16>()), Min<i16>()); - TestCast<i32>(ToString(Min<i32>()), Min<i32>()); - TestCast<i64>(ToString(Min<i64>()), Min<i64>()); - - // unsigned out of range - TestCast<ui8> (ToString(static_cast<ui64>(Max<ui8>()) + 1), Nothing()); - TestCast<ui16>(ToString(static_cast<ui64>(Max<ui16>()) + 1), Nothing()); - TestCast<ui32>(ToString(static_cast<ui64>(Max<ui32>()) + 1), Nothing()); - - // signed out of range (right) - TestCast<i8> (ToString(static_cast<i64>(Max<i8>()) + 1), Nothing()); - TestCast<i16>(ToString(static_cast<i64>(Max<i16>()) + 1), Nothing()); - TestCast<i32>(ToString(static_cast<i64>(Max<i32>()) + 1), Nothing()); - - // signed out of range (left) - TestCast<i8> (ToString(static_cast<i64>(Min<i8>()) - 1), Nothing()); - TestCast<i16>(ToString(static_cast<i64>(Min<i16>()) - 1), Nothing()); - TestCast<i32>(ToString(static_cast<i64>(Min<i32>()) - 1), Nothing()); - - // positive signed to unsigned - TestCast<ui8> (ToString(Max<i8>()), Max<i8>()); - TestCast<ui16>(ToString(Max<i16>()), Max<i16>()); - TestCast<ui32>(ToString(Max<i32>()), Max<i32>()); - TestCast<ui64>(ToString(Max<i64>()), Max<i64>()); - - // negative signed to unsigned - TestCast<ui8> (ToString(Min<i8>()), Nothing()); - TestCast<ui16>(ToString(Min<i16>()), Nothing()); - TestCast<ui32>(ToString(Min<i32>()), Nothing()); - TestCast<ui64>(ToString(Min<i64>()), Nothing()); - - // DyNumber limits - TestCast<ui64>("9.9999999999999999999999999999999999999E+125", Nothing()); - TestCast<ui64>("-9.9999999999999999999999999999999999999E+125", Nothing()); - TestCast<double>("1E-130", 1E-130); - TestCast<double>("-1E-130", -1E-130); - } + + Y_UNIT_TEST(Cast) { + TestCast<int>("0", 0); + + TestCast<int>("1", 1); + TestCast<int>("-1", -1); + + TestCast<int>("12", 12); + TestCast<int>("-12", -12); + TestCast<int>("123", 123); + TestCast<int>("-123", -123); + TestCast<int>("1234", 1234); + TestCast<int>("-1234", -1234); + + TestCast<int>(ToString(Max<int>()), Max<int>()); + TestCast<int>(ToString(Min<int>()), Min<int>()); + + TestCast<i8> ("200", Nothing()); + TestCast<i16>("40000", Nothing()); + TestCast<i32>("3000000000", Nothing()); + + TestCast<ui8> ("300", Nothing()); + TestCast<ui16>("70000", Nothing()); + TestCast<ui32>("5000000000", Nothing()); + + // int to floating point + TestCast<double>("1", 1); + TestCast<double>("12", 12); + TestCast<double>("123", 123); + + // floating point to int + TestCast<int>("0.1", Nothing()); + TestCast<int>("0.23", Nothing()); + TestCast<int>("1.2", Nothing()); + TestCast<int>("1.23", Nothing()); + TestCast<int>("12.3", Nothing()); + TestCast<int>("123.4", Nothing()); + + // double + TestCast<double>("0.1", 0.1); + TestCast<double>("0.23", 0.23); + TestCast<double>("-1.23", -1.23); + TestCast<double>("12.3", 12.3); + TestCast<double>("123.4", 123.4); + TestCast<double>("1.23E20", 1.23E20); + TestCast<double>("1.23E-20", 1.23E-20); + + // float + TestCast<float>("-0.1", -0.1f); + TestCast<float>("-0.23", -0.23f); + TestCast<float>("1.23", 1.23f); + TestCast<float>("-12.3", -12.3f); + TestCast<float>("-123.4", -123.4f); + TestCast<float>("-1.23E10", -1.23E10f); + TestCast<float>("-1.23E-10", -1.23E-10f); + + // unsigned Max + TestCast<ui8> (ToString(Max<ui8>()), Max<ui8>()); + TestCast<ui16>(ToString(Max<ui16>()), Max<ui16>()); + TestCast<ui32>(ToString(Max<ui32>()), Max<ui32>()); + TestCast<ui64>(ToString(Max<ui64>()), Max<ui64>()); + + // signed Max + TestCast<i8> (ToString(Max<i8>()), Max<i8>()); + TestCast<i16>(ToString(Max<i16>()), Max<i16>()); + TestCast<i32>(ToString(Max<i32>()), Max<i32>()); + TestCast<i64>(ToString(Max<i64>()), Max<i64>()); + + // signed Min + TestCast<i8> (ToString(Min<i8>()), Min<i8>()); + TestCast<i16>(ToString(Min<i16>()), Min<i16>()); + TestCast<i32>(ToString(Min<i32>()), Min<i32>()); + TestCast<i64>(ToString(Min<i64>()), Min<i64>()); + + // unsigned out of range + TestCast<ui8> (ToString(static_cast<ui64>(Max<ui8>()) + 1), Nothing()); + TestCast<ui16>(ToString(static_cast<ui64>(Max<ui16>()) + 1), Nothing()); + TestCast<ui32>(ToString(static_cast<ui64>(Max<ui32>()) + 1), Nothing()); + + // signed out of range (right) + TestCast<i8> (ToString(static_cast<i64>(Max<i8>()) + 1), Nothing()); + TestCast<i16>(ToString(static_cast<i64>(Max<i16>()) + 1), Nothing()); + TestCast<i32>(ToString(static_cast<i64>(Max<i32>()) + 1), Nothing()); + + // signed out of range (left) + TestCast<i8> (ToString(static_cast<i64>(Min<i8>()) - 1), Nothing()); + TestCast<i16>(ToString(static_cast<i64>(Min<i16>()) - 1), Nothing()); + TestCast<i32>(ToString(static_cast<i64>(Min<i32>()) - 1), Nothing()); + + // positive signed to unsigned + TestCast<ui8> (ToString(Max<i8>()), Max<i8>()); + TestCast<ui16>(ToString(Max<i16>()), Max<i16>()); + TestCast<ui32>(ToString(Max<i32>()), Max<i32>()); + TestCast<ui64>(ToString(Max<i64>()), Max<i64>()); + + // negative signed to unsigned + TestCast<ui8> (ToString(Min<i8>()), Nothing()); + TestCast<ui16>(ToString(Min<i16>()), Nothing()); + TestCast<ui32>(ToString(Min<i32>()), Nothing()); + TestCast<ui64>(ToString(Min<i64>()), Nothing()); + + // DyNumber limits + TestCast<ui64>("9.9999999999999999999999999999999999999E+125", Nothing()); + TestCast<ui64>("-9.9999999999999999999999999999999999999E+125", Nothing()); + TestCast<double>("1E-130", 1E-130); + TestCast<double>("-1E-130", -1E-130); + } } diff --git a/ydb/library/dynumber/ya.make b/ydb/library/dynumber/ya.make index ddecd3d527e..e125e312a7f 100644 --- a/ydb/library/dynumber/ya.make +++ b/ydb/library/dynumber/ya.make @@ -7,11 +7,11 @@ PEERDIR( ) SRCS( - cast.h + cast.h dynumber.cpp ) -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index 591e2e87615..edfcaa15302 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -50,20 +50,20 @@ "ListBase": "TCoNameValueTuple" }, { - "Name": "TCoChangefeed", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Name", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 2, "Name": "State", "Type": "TCoAtom"} - ] - }, - { - "Name": "TCoChangefeedList", - "ListBase": "TCoChangefeed" - }, - { + "Name": "TCoChangefeed", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Name", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 2, "Name": "State", "Type": "TCoAtom"} + ] + }, + { + "Name": "TCoChangefeedList", + "ListBase": "TCoChangefeed" + }, + { "Name": "TCoWorld", "Base": "TExprBase", "Match": {"Type": "NodeType", "TypeName": "World"}, diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index f7556882032..aa782ee93ff 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -158,7 +158,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TMaybeNode<TCoLambda> update; TVector<TCoNameValueTuple> other; TVector<TCoIndex> indexes; - TVector<TCoChangefeed> changefeeds; + TVector<TCoChangefeed> changefeeds; TMaybeNode<TExprList> columnFamilies; TVector<TCoNameValueTuple> tableSettings; TVector<TCoNameValueTuple> alterActions; @@ -206,23 +206,23 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } } indexes.push_back(index.Done()); - } else if (name == "changefeed") { - YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); - auto cf = Build<TCoChangefeed>(ctx, node.Pos()); - for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { - const auto& itemName = item.Name().Value(); - if (itemName == "name") { - cf.Name(item.Value().Cast<TCoAtom>()); - } else if (itemName == "settings") { - YQL_ENSURE(item.Value().Maybe<TCoNameValueTupleList>()); - cf.Settings(item.Value().Cast<TCoNameValueTupleList>()); - } else if (itemName == "state") { - cf.State(item.Value().Cast<TCoAtom>()); - } else { - YQL_ENSURE(false, "unknown changefeed item"); - } - } - changefeeds.push_back(cf.Done()); + } else if (name == "changefeed") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + auto cf = Build<TCoChangefeed>(ctx, node.Pos()); + for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { + const auto& itemName = item.Name().Value(); + if (itemName == "name") { + cf.Name(item.Value().Cast<TCoAtom>()); + } else if (itemName == "settings") { + YQL_ENSURE(item.Value().Maybe<TCoNameValueTupleList>()); + cf.Settings(item.Value().Cast<TCoNameValueTupleList>()); + } else if (itemName == "state") { + cf.State(item.Value().Cast<TCoAtom>()); + } else { + YQL_ENSURE(false, "unknown changefeed item"); + } + } + changefeeds.push_back(cf.Done()); } else if (name == "columnFamilies") { YQL_ENSURE(tuple.Value().Maybe<TExprList>()); columnFamilies = tuple.Value().Cast<TExprList>(); @@ -250,10 +250,10 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { .Add(indexes) .Done(); - const auto& cfs = Build<TCoChangefeedList>(ctx, node.Pos()) - .Add(changefeeds) - .Done(); - + const auto& cfs = Build<TCoChangefeedList>(ctx, node.Pos()) + .Add(changefeeds) + .Done(); + const auto& tableProfileSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) .Add(tableSettings) .Done(); @@ -275,7 +275,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.Filter = filter; ret.Update = update; ret.Indexes = idx; - ret.Changefeeds = cfs; + ret.Changefeeds = cfs; ret.ColumnFamilies = columnFamilies; ret.TableSettings = tableProfileSettings; ret.AlterActions = alterTableActions; diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index 01d419ae064..9e80cfd1355 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -41,7 +41,7 @@ struct TWriteTableSettings { NNodes::TMaybeNode<NNodes::TCoLambda> Filter; NNodes::TMaybeNode<NNodes::TCoLambda> Update; NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes; - NNodes::TMaybeNode<NNodes::TCoChangefeedList> Changefeeds; + NNodes::TMaybeNode<NNodes::TCoChangefeedList> Changefeeds; NNodes::TCoNameValueTupleList Other; NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies; NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings; diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 90862d4d45b..ef394c7a009 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -450,14 +450,14 @@ simple_values_source: expr_list | select_stmt; create_table_stmt: CREATE TABLE simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN with_table_settings?; -create_table_entry: - column_schema - | table_constraint - | table_index - | family_entry - | changefeed -; - +create_table_entry: + column_schema + | table_constraint + | table_index + | family_entry + | changefeed +; + with_table_settings: WITH LPAREN table_settings_entry (COMMA table_settings_entry)* RPAREN; table_settings_entry: an_id EQUALS table_setting_value; @@ -468,31 +468,31 @@ alter_table_action: | alter_table_alter_column | alter_table_add_column_family | alter_table_alter_column_family - | alter_table_set_table_setting_uncompat - | alter_table_set_table_setting_compat - | alter_table_reset_table_setting + | alter_table_set_table_setting_uncompat + | alter_table_set_table_setting_compat + | alter_table_reset_table_setting | alter_table_add_index | alter_table_drop_index | alter_table_rename_to - | alter_table_add_changefeed - | alter_table_alter_changefeed - | alter_table_drop_changefeed -; + | alter_table_add_changefeed + | alter_table_alter_changefeed + | alter_table_drop_changefeed +; alter_table_add_column: ADD COLUMN? column_schema; alter_table_drop_column: DROP COLUMN? an_id; alter_table_alter_column: ALTER COLUMN an_id SET family_relation; alter_table_add_column_family: ADD family_entry; alter_table_alter_column_family: ALTER FAMILY an_id SET an_id family_setting_value; -alter_table_set_table_setting_uncompat: SET an_id table_setting_value; -alter_table_set_table_setting_compat: SET LPAREN alter_table_setting_entry (COMMA alter_table_setting_entry)* RPAREN; -alter_table_reset_table_setting: RESET LPAREN an_id (COMMA an_id)* RPAREN; +alter_table_set_table_setting_uncompat: SET an_id table_setting_value; +alter_table_set_table_setting_compat: SET LPAREN alter_table_setting_entry (COMMA alter_table_setting_entry)* RPAREN; +alter_table_reset_table_setting: RESET LPAREN an_id (COMMA an_id)* RPAREN; alter_table_add_index: ADD table_index; alter_table_drop_index: DROP INDEX an_id; alter_table_rename_to: RENAME TO an_id_table; -alter_table_add_changefeed: ADD changefeed; -alter_table_alter_changefeed: ALTER CHANGEFEED an_id changefeed_alter_settings; -alter_table_drop_changefeed: DROP CHANGEFEED an_id; +alter_table_add_changefeed: ADD changefeed; +alter_table_alter_changefeed: ALTER CHANGEFEED an_id changefeed_alter_settings; +alter_table_drop_changefeed: DROP CHANGEFEED an_id; column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; family_relation: FAMILY an_id; @@ -504,36 +504,36 @@ table_constraint: | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN ; -table_index: INDEX an_id table_index_type +table_index: INDEX an_id table_index_type (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? ON LPAREN an_id_schema (COMMA an_id_schema)* RPAREN (COVER LPAREN an_id_schema (COMMA an_id_schema)* RPAREN)?; -table_index_type: - global_index - | local_index -; - -global_index: GLOBAL UNIQUE? (SYNC | ASYNC)?; -local_index: LOCAL; - -changefeed: CHANGEFEED an_id WITH LPAREN changefeed_settings RPAREN; -changefeed_settings: changefeed_settings_entry (COMMA changefeed_settings_entry)*; -changefeed_settings_entry: an_id EQUALS changefeed_setting_value; -changefeed_setting_value: STRING_VALUE; -changefeed_alter_settings: - DISABLE - | SET LPAREN changefeed_settings RPAREN -; - -alter_table_setting_entry: an_id EQUALS table_setting_value; - +table_index_type: + global_index + | local_index +; + +global_index: GLOBAL UNIQUE? (SYNC | ASYNC)?; +local_index: LOCAL; + +changefeed: CHANGEFEED an_id WITH LPAREN changefeed_settings RPAREN; +changefeed_settings: changefeed_settings_entry (COMMA changefeed_settings_entry)*; +changefeed_settings_entry: an_id EQUALS changefeed_setting_value; +changefeed_setting_value: STRING_VALUE; +changefeed_alter_settings: + DISABLE + | SET LPAREN changefeed_settings RPAREN +; + +alter_table_setting_entry: an_id EQUALS table_setting_value; + table_setting_value: id | STRING_VALUE | integer | split_boundaries - | expr ON an_id + | expr ON an_id ; family_entry: FAMILY an_id family_settings?; @@ -865,7 +865,7 @@ keyword_compat: ( | ANSI | ARRAY | ASC - | ASYNC + | ASYNC | ATTACH | AUTOINCREMENT | BEFORE @@ -873,7 +873,7 @@ keyword_compat: ( | BERNOULLI | BY | CASCADE - | CHANGEFEED + | CHANGEFEED | CHECK | COLLATE | COMMIT @@ -894,7 +894,7 @@ keyword_compat: ( | DELETE | DESC | DETACH - | DISABLE + | DISABLE | DISCARD | DO | DROP @@ -972,7 +972,7 @@ keyword_compat: ( | RELEASE | RENAME | REPLACE - | RESET + | RESET | RESPECT | RESTRICT | RESULT @@ -989,7 +989,7 @@ keyword_compat: ( | SORTED | SUBQUERY | SYMBOLS - | SYNC + | SYNC | SYSTEM | TABLE | TABLESAMPLE @@ -1125,7 +1125,7 @@ ARRAY: A R R A Y; AS: A S; ASC: A S C; ASSUME: A S S U M E; -ASYNC: A S Y N C; +ASYNC: A S Y N C; ATTACH: A T T A C H; AUTOINCREMENT: A U T O I N C R E M E N T; AUTOMAP: A U T O M A P; @@ -1139,7 +1139,7 @@ CALLABLE: C A L L A B L E; CASCADE: C A S C A D E; CASE: C A S E; CAST: C A S T; -CHANGEFEED: C H A N G E F E E D; +CHANGEFEED: C H A N G E F E E D; CHECK: C H E C K; COLLATE: C O L L A T E; COLUMN: C O L U M N; @@ -1168,7 +1168,7 @@ DELETE: D E L E T E; DESC: D E S C; DETACH: D E T A C H; DICT: D I C T; -DISABLE: D I S A B L E; +DISABLE: D I S A B L E; DISCARD: D I S C A R D; DISTINCT: D I S T I N C T; DO: D O; @@ -1274,7 +1274,7 @@ RELEASE: R E L E A S E; RENAME: R E N A M E; REPEATABLE: R E P E A T A B L E; REPLACE: R E P L A C E; -RESET: R E S E T; +RESET: R E S E T; RESOURCE: R E S O U R C E; RESPECT: R E S P E C T; RESTRICT: R E S T R I C T; @@ -1300,7 +1300,7 @@ STREAM: S T R E A M; STRUCT: S T R U C T; SUBQUERY: S U B Q U E R Y; SYMBOLS: S Y M B O L S; -SYNC: S Y N C; +SYNC: S Y N C; SYSTEM: S Y S T E M; TABLE: T A B L E; TABLESAMPLE: T A B L E S A M P L E; diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index 336f0777660..870bbd9cc7f 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -2222,12 +2222,12 @@ TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, co (ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default); } -TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr) - : ColumnName(columnName) - , Expr(expr) -{ -} - +TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr) + : ColumnName(columnName) + , Expr(expr) +{ +} + TString IdContent(TContext& ctx, const TString& s) { YQL_ENSURE(!s.empty(), "Empty identifier not expected"); if (!s.StartsWith('`')) { diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 3f65b3ddb36..de82b45a238 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -18,7 +18,7 @@ #include <array> #include <functional> -#include <variant> +#include <variant> namespace NSQLTranslationV1 { constexpr const size_t SQL_MAX_INLINE_SCRIPT_LEN = 24; @@ -1052,13 +1052,13 @@ namespace NSQLTranslationV1 { TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input); TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input); - struct TTtlSettings { - TIdentifier ColumnName; - TNodePtr Expr; - - TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr); - }; - + struct TTtlSettings { + TIdentifier ColumnName; + TNodePtr Expr; + + TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr); + }; + struct TTableSettings { TNodePtr CompactionPolicy; TMaybe<TIdentifier> AutoPartitioningBySize; @@ -1070,12 +1070,12 @@ namespace NSQLTranslationV1 { TVector<TVector<TNodePtr>> PartitionAtKeys; TMaybe<TIdentifier> KeyBloomFilter; TNodePtr ReadReplicasSettings; - NYql::TResetableSetting<TTtlSettings, void> TtlSettings; - + NYql::TResetableSetting<TTtlSettings, void> TtlSettings; + bool IsSet() const { return CompactionPolicy || AutoPartitioningBySize || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || UniformPartitions || PartitionAtKeys || KeyBloomFilter - || ReadReplicasSettings || TtlSettings; + || ReadReplicasSettings || TtlSettings; } }; @@ -1089,52 +1089,52 @@ namespace NSQLTranslationV1 { TNodePtr Compression; }; - struct TIndexDescription { - enum class EType { - GlobalSync, - GlobalAsync, - }; - - TIndexDescription(const TIdentifier& name, EType type = EType::GlobalSync) - : Name(name) - , Type(type) - {} - - TIdentifier Name; - EType Type; - TVector<TIdentifier> IndexColumns; - TVector<TIdentifier> DataColumns; - }; - - struct TChangefeedSettings { - struct TLocalSinkSettings { - // no special settings - }; - - TNodePtr Mode; - TNodePtr Format; - std::optional<std::variant<TLocalSinkSettings>> SinkSettings; - }; - - struct TChangefeedDescription { - TChangefeedDescription(const TIdentifier& name) - : Name(name) - , Disable(false) - {} - - TIdentifier Name; - TChangefeedSettings Settings; - bool Disable; - }; - + struct TIndexDescription { + enum class EType { + GlobalSync, + GlobalAsync, + }; + + TIndexDescription(const TIdentifier& name, EType type = EType::GlobalSync) + : Name(name) + , Type(type) + {} + + TIdentifier Name; + EType Type; + TVector<TIdentifier> IndexColumns; + TVector<TIdentifier> DataColumns; + }; + + struct TChangefeedSettings { + struct TLocalSinkSettings { + // no special settings + }; + + TNodePtr Mode; + TNodePtr Format; + std::optional<std::variant<TLocalSinkSettings>> SinkSettings; + }; + + struct TChangefeedDescription { + TChangefeedDescription(const TIdentifier& name) + : Name(name) + , Disable(false) + {} + + TIdentifier Name; + TChangefeedSettings Settings; + bool Disable; + }; + struct TCreateTableParameters { TVector<TColumnSchema> Columns; TVector<TIdentifier> PkColumns; TVector<TIdentifier> PartitionByColumns; TVector<std::pair<TIdentifier, bool>> OrderByColumns; - TVector<TIndexDescription> Indexes; + TVector<TIndexDescription> Indexes; TVector<TFamilyEntry> ColumnFamilies; - TVector<TChangefeedDescription> Changefeeds; + TVector<TChangefeedDescription> Changefeeds; TTableSettings TableSettings; }; @@ -1145,20 +1145,20 @@ namespace NSQLTranslationV1 { TVector<TFamilyEntry> AddColumnFamilies; TVector<TFamilyEntry> AlterColumnFamilies; TTableSettings TableSettings; - TVector<TIndexDescription> AddIndexes; + TVector<TIndexDescription> AddIndexes; TVector<TIdentifier> DropIndexes; TMaybe<TIdentifier> RenameTo; - TVector<TChangefeedDescription> AddChangefeeds; - TVector<TChangefeedDescription> AlterChangefeeds; - TVector<TIdentifier> DropChangefeeds; + TVector<TChangefeedDescription> AddChangefeeds; + TVector<TChangefeedDescription> AlterChangefeeds; + TVector<TIdentifier> DropChangefeeds; bool IsEmpty() const { return AddColumns.empty() && DropColumns.empty() && AlterColumns.empty() - && AddColumnFamilies.empty() && AlterColumnFamilies.empty() - && !TableSettings.IsSet() - && AddIndexes.empty() && DropIndexes.empty() - && !RenameTo.Defined() - && AddChangefeeds.empty() && AlterChangefeeds.empty() && DropChangefeeds.empty(); + && AddColumnFamilies.empty() && AlterColumnFamilies.empty() + && !TableSettings.IsSet() + && AddIndexes.empty() && DropIndexes.empty() + && !RenameTo.Defined() + && AddChangefeeds.empty() && AlterChangefeeds.empty() && DropChangefeeds.empty(); } }; diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 8e4311b5e4c..725356b9f57 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -89,62 +89,62 @@ TNodePtr BuildTableKey(TPosition pos, const TString& service, const TDeferredAto return new TUniqueTableKey(pos, service, cluster, name, view); } -static INode::TPtr CreateIndexType(TIndexDescription::EType type, const INode& node) { - switch (type) { - case TIndexDescription::EType::GlobalSync: - return node.Q("syncGlobal"); - case TIndexDescription::EType::GlobalAsync: - return node.Q("asyncGlobal"); - } -} - -static INode::TPtr CreateIndexDesc(const TIndexDescription& index, const INode& node) { +static INode::TPtr CreateIndexType(TIndexDescription::EType type, const INode& node) { + switch (type) { + case TIndexDescription::EType::GlobalSync: + return node.Q("syncGlobal"); + case TIndexDescription::EType::GlobalAsync: + return node.Q("asyncGlobal"); + } +} + +static INode::TPtr CreateIndexDesc(const TIndexDescription& index, const INode& node) { auto indexColumns = node.Y(); - for (const auto& col : index.IndexColumns) { + for (const auto& col : index.IndexColumns) { indexColumns = node.L(indexColumns, BuildQuotedAtom(col.Pos, col.Name)); } auto dataColumns = node.Y(); - for (const auto& col : index.DataColumns) { + for (const auto& col : index.DataColumns) { dataColumns = node.L(dataColumns, BuildQuotedAtom(col.Pos, col.Name)); } - const auto& indexType = node.Y(node.Q("indexType"), CreateIndexType(index.Type, node)); - const auto& indexName = node.Y(node.Q("indexName"), BuildQuotedAtom(index.Name.Pos, index.Name.Name)); + const auto& indexType = node.Y(node.Q("indexType"), CreateIndexType(index.Type, node)); + const auto& indexName = node.Y(node.Q("indexName"), BuildQuotedAtom(index.Name.Pos, index.Name.Name)); return node.Y(node.Q(indexName), node.Q(indexType), node.Q(node.Y(node.Q("indexColumns"), node.Q(indexColumns))), node.Q(node.Y(node.Q("dataColumns"), node.Q(dataColumns)))); } -static INode::TPtr CreateChangefeedDesc(const TChangefeedDescription& desc, const INode& node) { - auto settings = node.Y(); - if (desc.Settings.Mode) { - settings = node.L(settings, node.Q(node.Y(node.Q("mode"), desc.Settings.Mode))); - } - if (desc.Settings.Format) { - settings = node.L(settings, node.Q(node.Y(node.Q("format"), desc.Settings.Format))); - } - if (const auto& sink = desc.Settings.SinkSettings) { - switch (sink->index()) { - case 0: // local - settings = node.L(settings, node.Q(node.Y(node.Q("local"), node.Q(node.Y())))); - break; - default: - YQL_ENSURE(false, "Unexpected sink settings"); - } - } - - auto state = node.Y(); - if (desc.Disable) { - state = node.Q("disable"); - } - - return node.Y( - node.Q(node.Y(node.Q("name"), BuildQuotedAtom(desc.Name.Pos, desc.Name.Name))), - node.Q(node.Y(node.Q("settings"), node.Q(settings))), - node.Q(node.Y(node.Q("state"), node.Q(state))) - ); -} - +static INode::TPtr CreateChangefeedDesc(const TChangefeedDescription& desc, const INode& node) { + auto settings = node.Y(); + if (desc.Settings.Mode) { + settings = node.L(settings, node.Q(node.Y(node.Q("mode"), desc.Settings.Mode))); + } + if (desc.Settings.Format) { + settings = node.L(settings, node.Q(node.Y(node.Q("format"), desc.Settings.Format))); + } + if (const auto& sink = desc.Settings.SinkSettings) { + switch (sink->index()) { + case 0: // local + settings = node.L(settings, node.Q(node.Y(node.Q("local"), node.Q(node.Y())))); + break; + default: + YQL_ENSURE(false, "Unexpected sink settings"); + } + } + + auto state = node.Y(); + if (desc.Disable) { + state = node.Q("disable"); + } + + return node.Y( + node.Q(node.Y(node.Q("name"), BuildQuotedAtom(desc.Name.Pos, desc.Name.Name))), + node.Q(node.Y(node.Q("settings"), node.Q(settings))), + node.Q(node.Y(node.Q("state"), node.Q(state))) + ); +} + class TPrepTableKeys: public ITableKeys { public: TPrepTableKeys(TPosition pos, const TString& service, const TDeferredAtom& cluster, @@ -577,12 +577,12 @@ public: return false; } - if (!Params.PkColumns.empty() - || !Params.PartitionByColumns.empty() - || !Params.OrderByColumns.empty() - || !Params.Indexes.empty() - || !Params.Changefeeds.empty()) - { + if (!Params.PkColumns.empty() + || !Params.PartitionByColumns.empty() + || !Params.OrderByColumns.empty() + || !Params.Indexes.empty() + || !Params.Changefeeds.empty()) + { THashSet<TString> columnsSet; for (auto& col : Params.Columns) { columnsSet.insert(col.Name); @@ -608,34 +608,34 @@ public: } THashSet<TString> indexNames; - for (const auto& index : Params.Indexes) { - if (!indexNames.insert(index.Name.Name).second) { - ctx.Error(index.Name.Pos) << "Index " << index.Name.Name << " must be defined once"; + for (const auto& index : Params.Indexes) { + if (!indexNames.insert(index.Name.Name).second) { + ctx.Error(index.Name.Pos) << "Index " << index.Name.Name << " must be defined once"; return false; } - for (const auto& indexColumn : index.IndexColumns) { + for (const auto& indexColumn : index.IndexColumns) { if (!columnsSet.contains(indexColumn.Name)) { ctx.Error(indexColumn.Pos) << "Undefined column: " << indexColumn.Name; return false; } } - for (const auto& dataColumn : index.DataColumns) { + for (const auto& dataColumn : index.DataColumns) { if (!columnsSet.contains(dataColumn.Name)) { ctx.Error(dataColumn.Pos) << "Undefined column: " << dataColumn.Name; return false; } } } - - THashSet<TString> cfNames; - for (const auto& cf : Params.Changefeeds) { - if (!cfNames.insert(cf.Name.Name).second) { - ctx.Error(cf.Name.Pos) << "Changefeed " << cf.Name.Name << " must be defined once"; - return false; - } - } + + THashSet<TString> cfNames; + for (const auto& cf : Params.Changefeeds) { + if (!cfNames.insert(cf.Name.Name).second) { + ctx.Error(cf.Name.Pos) << "Changefeed " << cf.Name.Name << " must be defined once"; + return false; + } + } } auto columns = Y(); for (auto& col : Params.Columns) { @@ -706,16 +706,16 @@ public: opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); } - for (const auto& index : Params.Indexes) { - const auto& desc = CreateIndexDesc(index, *this); + for (const auto& index : Params.Indexes) { + const auto& desc = CreateIndexDesc(index, *this); opts = L(opts, Q(Y(Q("index"), Q(desc)))); } - for (const auto& cf : Params.Changefeeds) { - const auto& desc = CreateChangefeedDesc(cf, *this); - opts = L(opts, Q(Y(Q("changefeed"), Q(desc)))); - } - + for (const auto& cf : Params.Changefeeds) { + const auto& desc = CreateChangefeedDesc(cf, *this); + opts = L(opts, Q(Y(Q("changefeed"), Q(desc)))); + } + if (Params.ColumnFamilies) { auto columnFamilies = Y(); for (const auto& family : Params.ColumnFamilies) { @@ -776,19 +776,19 @@ public: if (Params.TableSettings.ReadReplicasSettings) { settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); } - if (const auto& ttl = Params.TableSettings.TtlSettings) { - if (ttl.IsSet()) { - const auto& ttlSettings = ttl.GetValueSet(); - auto columnName = BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name); - auto nameValueTuple = Y( - Q(Y(Q("columnName"), columnName)), - Q(Y(Q("expireAfter"), ttlSettings.Expr)) - ); - settings = L(settings, Q(Y(Q("setTtlSettings"), Q(nameValueTuple)))); - } else { - YQL_ENSURE(false, "Can't reset TTL settings"); - } - } + if (const auto& ttl = Params.TableSettings.TtlSettings) { + if (ttl.IsSet()) { + const auto& ttlSettings = ttl.GetValueSet(); + auto columnName = BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name); + auto nameValueTuple = Y( + Q(Y(Q("columnName"), columnName)), + Q(Y(Q("expireAfter"), ttlSettings.Expr)) + ); + settings = L(settings, Q(Y(Q("setTtlSettings"), Q(nameValueTuple)))); + } else { + YQL_ENSURE(false, "Can't reset TTL settings"); + } + } opts = L(opts, Q(Y(Q("tableSettings"), Q(settings)))); } @@ -941,24 +941,24 @@ public: if (Params.TableSettings.ReadReplicasSettings) { settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); } - if (const auto& ttl = Params.TableSettings.TtlSettings) { - if (ttl.IsSet()) { - const auto& ttlSettings = ttl.GetValueSet(); - auto columnName = BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name); - auto nameValueTuple = Y( - Q(Y(Q("columnName"), columnName)), - Q(Y(Q("expireAfter"), ttlSettings.Expr)) - ); - settings = L(settings, Q(Y(Q("setTtlSettings"), Q(nameValueTuple)))); - } else { - settings = L(settings, Q(Y(Q("resetTtlSettings"), Q(Y())))); - } - } + if (const auto& ttl = Params.TableSettings.TtlSettings) { + if (ttl.IsSet()) { + const auto& ttlSettings = ttl.GetValueSet(); + auto columnName = BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name); + auto nameValueTuple = Y( + Q(Y(Q("columnName"), columnName)), + Q(Y(Q("expireAfter"), ttlSettings.Expr)) + ); + settings = L(settings, Q(Y(Q("setTtlSettings"), Q(nameValueTuple)))); + } else { + settings = L(settings, Q(Y(Q("resetTtlSettings"), Q(Y())))); + } + } actions = L(actions, Q(Y(Q("setTableSettings"), Q(settings)))); } - for (const auto& index : Params.AddIndexes) { - const auto& desc = CreateIndexDesc(index, *this); + for (const auto& index : Params.AddIndexes) { + const auto& desc = CreateIndexDesc(index, *this); actions = L(actions, Q(Y(Q("addIndex"), Q(desc)))); } @@ -973,21 +973,21 @@ public: actions = L(actions, Q(Y(Q("renameTo"), destination))); } - for (const auto& cf : Params.AddChangefeeds) { - const auto& desc = CreateChangefeedDesc(cf, *this); - actions = L(actions, Q(Y(Q("addChangefeed"), Q(desc)))); - } - - for (const auto& cf : Params.AlterChangefeeds) { - const auto& desc = CreateChangefeedDesc(cf, *this); - actions = L(actions, Q(Y(Q("alterChangefeed"), Q(desc)))); - } - - for (const auto& id : Params.DropChangefeeds) { - const auto name = BuildQuotedAtom(id.Pos, id.Name); - actions = L(actions, Q(Y(Q("dropChangefeed"), name))); - } - + for (const auto& cf : Params.AddChangefeeds) { + const auto& desc = CreateChangefeedDesc(cf, *this); + actions = L(actions, Q(Y(Q("addChangefeed"), Q(desc)))); + } + + for (const auto& cf : Params.AlterChangefeeds) { + const auto& desc = CreateChangefeedDesc(cf, *this); + actions = L(actions, Q(Y(Q("alterChangefeed"), Q(desc)))); + } + + for (const auto& id : Params.DropChangefeeds) { + const auto name = BuildQuotedAtom(id.Pos, id.Name); + actions = L(actions, Q(Y(Q("dropChangefeed"), name))); + } + auto opts = Y(); opts = L(opts, Q(Y(Q("mode"), Q("alter")))); diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 6a37c7e89ad..ab5a8647885 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -596,30 +596,30 @@ static bool PureColumnOrNamedListStr(const TRule_pure_column_or_named_list& node return true; } -static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, TVector<TIndexDescription>& indexes) { - indexes.emplace_back(IdEx(node.GetRule_an_id2(), ctx)); +static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, TVector<TIndexDescription>& indexes) { + indexes.emplace_back(IdEx(node.GetRule_an_id2(), ctx)); - const auto& indexType = node.GetRule_table_index_type3(); + const auto& indexType = node.GetRule_table_index_type3(); switch (indexType.Alt_case()) { - case TRule_table_index_type::kAltTableIndexType1: { - auto globalIndex = indexType.GetAlt_table_index_type1().GetRule_global_index1(); - if (globalIndex.HasBlock2()) { + case TRule_table_index_type::kAltTableIndexType1: { + auto globalIndex = indexType.GetAlt_table_index_type1().GetRule_global_index1(); + if (globalIndex.HasBlock2()) { ctx.AltNotImplemented("unique", indexType); return false; } - if (globalIndex.HasBlock3()) { - const TString token = to_lower(ctx.Token(globalIndex.GetBlock3().GetToken1())); - if (token == "sync") { - indexes.back().Type = TIndexDescription::EType::GlobalSync; - } else if (token == "async") { - indexes.back().Type = TIndexDescription::EType::GlobalAsync; - } else { - Y_FAIL("You should change implementation according to grammar changes"); - } - } + if (globalIndex.HasBlock3()) { + const TString token = to_lower(ctx.Token(globalIndex.GetBlock3().GetToken1())); + if (token == "sync") { + indexes.back().Type = TIndexDescription::EType::GlobalSync; + } else if (token == "async") { + indexes.back().Type = TIndexDescription::EType::GlobalAsync; + } else { + Y_FAIL("You should change implementation according to grammar changes"); + } + } } break; - case TRule_table_index_type::kAltTableIndexType2: + case TRule_table_index_type::kAltTableIndexType2: ctx.AltNotImplemented("local", indexType); return false; default: @@ -631,75 +631,75 @@ static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, T return false; } - indexes.back().IndexColumns.emplace_back(IdEx(node.GetRule_an_id_schema7(), ctx)); + indexes.back().IndexColumns.emplace_back(IdEx(node.GetRule_an_id_schema7(), ctx)); for (const auto& block : node.GetBlock8()) { - indexes.back().IndexColumns.emplace_back(IdEx(block.GetRule_an_id_schema2(), ctx)); + indexes.back().IndexColumns.emplace_back(IdEx(block.GetRule_an_id_schema2(), ctx)); } if (node.HasBlock10()) { const auto& block = node.GetBlock10(); - indexes.back().DataColumns.emplace_back(IdEx(block.GetRule_an_id_schema3(), ctx)); + indexes.back().DataColumns.emplace_back(IdEx(block.GetRule_an_id_schema3(), ctx)); for (const auto& inner : block.GetBlock4()) { - indexes.back().DataColumns.emplace_back(IdEx(inner.GetRule_an_id_schema2(), ctx)); + indexes.back().DataColumns.emplace_back(IdEx(inner.GetRule_an_id_schema2(), ctx)); + } + } + + return true; +} + +static bool ChangefeedSettingsEntry(const TRule_changefeed_settings_entry& node, TTranslation& ctx, TChangefeedSettings& settings, bool alter) { + const auto id = IdEx(node.GetRule_an_id1(), ctx); + const TString value(ctx.Token(node.GetRule_changefeed_setting_value3().GetToken1())); + + if (alter) { + // currently we don't support alter settings + ctx.Error() << to_upper(id.Name) << " alter is not supported"; + return false; + } + + if (to_lower(id.Name) == "sink_type") { + if (to_lower(value) == "local") { + settings.SinkSettings = TChangefeedSettings::TLocalSinkSettings(); + } else { + ctx.Error() << "Unknown changefeed's sink type: " << to_upper(value); + return false; + } + } else if (to_lower(id.Name) == "mode") { + settings.Mode = BuildLiteralSmartString(ctx.Context(), value); + } else if (to_lower(id.Name) == "format") { + settings.Format = BuildLiteralSmartString(ctx.Context(), value); + } else { + ctx.Error() << "Unknown changefeed setting: " << id.Name; + return false; + } + + return true; +} + +static bool ChangefeedSettings(const TRule_changefeed_settings& node, TTranslation& ctx, TChangefeedSettings& settings, bool alter) { + if (!ChangefeedSettingsEntry(node.GetRule_changefeed_settings_entry1(), ctx, settings, alter)) { + return false; + } + + for (auto& block : node.GetBlock2()) { + if (!ChangefeedSettingsEntry(block.GetRule_changefeed_settings_entry2(), ctx, settings, alter)) { + return false; } } return true; } -static bool ChangefeedSettingsEntry(const TRule_changefeed_settings_entry& node, TTranslation& ctx, TChangefeedSettings& settings, bool alter) { - const auto id = IdEx(node.GetRule_an_id1(), ctx); - const TString value(ctx.Token(node.GetRule_changefeed_setting_value3().GetToken1())); - - if (alter) { - // currently we don't support alter settings - ctx.Error() << to_upper(id.Name) << " alter is not supported"; - return false; - } - - if (to_lower(id.Name) == "sink_type") { - if (to_lower(value) == "local") { - settings.SinkSettings = TChangefeedSettings::TLocalSinkSettings(); - } else { - ctx.Error() << "Unknown changefeed's sink type: " << to_upper(value); - return false; - } - } else if (to_lower(id.Name) == "mode") { - settings.Mode = BuildLiteralSmartString(ctx.Context(), value); - } else if (to_lower(id.Name) == "format") { - settings.Format = BuildLiteralSmartString(ctx.Context(), value); - } else { - ctx.Error() << "Unknown changefeed setting: " << id.Name; - return false; - } - - return true; -} - -static bool ChangefeedSettings(const TRule_changefeed_settings& node, TTranslation& ctx, TChangefeedSettings& settings, bool alter) { - if (!ChangefeedSettingsEntry(node.GetRule_changefeed_settings_entry1(), ctx, settings, alter)) { - return false; - } - - for (auto& block : node.GetBlock2()) { - if (!ChangefeedSettingsEntry(block.GetRule_changefeed_settings_entry2(), ctx, settings, alter)) { - return false; - } - } - - return true; -} - -static bool CreateChangefeed(const TRule_changefeed& node, TTranslation& ctx, TVector<TChangefeedDescription>& changefeeds) { - changefeeds.emplace_back(IdEx(node.GetRule_an_id2(), ctx)); - - if (!ChangefeedSettings(node.GetRule_changefeed_settings5(), ctx, changefeeds.back().Settings, false)) { - return false; - } - - return true; -} - +static bool CreateChangefeed(const TRule_changefeed& node, TTranslation& ctx, TVector<TChangefeedDescription>& changefeeds) { + changefeeds.emplace_back(IdEx(node.GetRule_an_id2(), ctx)); + + if (!ChangefeedSettings(node.GetRule_changefeed_settings5(), ctx, changefeeds.back().Settings, false)) { + return false; + } + + return true; +} + static std::pair<TString, TString> TableKeyImpl(const std::pair<bool, TString>& nameWithAt, TString view, TTranslation& ctx) { if (nameWithAt.first) { view = "@"; @@ -845,11 +845,11 @@ protected: bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family); bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family); bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings); - bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, - bool alter, bool reset); + bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, + bool alter, bool reset); bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, bool alter = false); - bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings); + bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings); TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); TNodePtr TypeDecimal(const TRule_type_name_decimal& node); @@ -1948,15 +1948,15 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr params.ColumnFamilies.push_back(family); break; } - case TRule_create_table_entry::kAltCreateTableEntry5: - { - // changefeed - auto& changefeed = node.GetAlt_create_table_entry5().GetRule_changefeed1(); - if (!CreateChangefeed(changefeed, *this, params.Changefeeds)) { - return false; - } - break; - } + case TRule_create_table_entry::kAltCreateTableEntry5: + { + // changefeed + auto& changefeed = node.GetAlt_create_table_entry5().GetRule_changefeed1(); + if (!CreateChangefeed(changefeed, *this, params.Changefeeds)) { + return false; + } + break; + } default: AltNotImplemented("create_table_entry", node); return false; @@ -2098,30 +2098,30 @@ namespace { } return true; } - - bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, - TSqlExpression& expr, TContext& ctx, TTranslation& txc) { - switch (from.Alt_case()) { - case TRule_table_setting_value::kAltTableSettingValue5: { - auto columnName = IdEx(from.GetAlt_table_setting_value5().GetRule_an_id3(), txc); - auto exprNode = expr.Build(from.GetAlt_table_setting_value5().GetRule_expr1()); - if (!exprNode) { - return false; - } - - if (exprNode->GetOpName() != "Interval") { - ctx.Error() << "Literal of Interval type is expected for TTL"; - return false; - } - - to.Set(TTtlSettings(columnName, exprNode)); - break; - } - default: - return false; - } - return true; - } + + bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, + TSqlExpression& expr, TContext& ctx, TTranslation& txc) { + switch (from.Alt_case()) { + case TRule_table_setting_value::kAltTableSettingValue5: { + auto columnName = IdEx(from.GetAlt_table_setting_value5().GetRule_an_id3(), txc); + auto exprNode = expr.Build(from.GetAlt_table_setting_value5().GetRule_expr1()); + if (!exprNode) { + return false; + } + + if (exprNode->GetOpName() != "Interval") { + ctx.Error() << "Literal of Interval type is expected for TTL"; + return false; + } + + to.Set(TTtlSettings(columnName, exprNode)); + break; + } + default: + return false; + } + return true; + } bool WithoutAlpha(const std::string_view& literal) { return literal.cend() == std::find_if(literal.cbegin(), literal.cend(), [](char c) { return std::isalpha(c) || (c & '\x80'); }); @@ -2170,113 +2170,113 @@ namespace { } } -bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, - TTableSettings& settings, bool alter, bool reset) { - YQL_ENSURE(value || reset); - YQL_ENSURE(!reset || reset & alter); - +bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, + TTableSettings& settings, bool alter, bool reset) { + YQL_ENSURE(value || reset); + YQL_ENSURE(!reset || reset & alter); + if (to_lower(id.Name) == "compaction_policy") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreString(*value, settings.CompactionPolicy, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; return false; } - if (!StoreString(*value, settings.CompactionPolicy, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; - return false; - } } else if (to_lower(id.Name) == "auto_partitioning_by_size") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreId(*value, settings.AutoPartitioningBySize, *this)) { + Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - if (!StoreId(*value, settings.AutoPartitioningBySize, *this)) { - Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; - return false; - } } else if (to_lower(id.Name) == "auto_partitioning_partition_size_mb") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreInt(*value, settings.PartitionSizeMb, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - if (!StoreInt(*value, settings.PartitionSizeMb, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be an integer"; - return false; - } } else if (to_lower(id.Name) == "auto_partitioning_by_load") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreId(*value, settings.AutoPartitioningByLoad, *this)) { + Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - if (!StoreId(*value, settings.AutoPartitioningByLoad, *this)) { - Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; - return false; - } } else if (to_lower(id.Name) == "auto_partitioning_min_partitions_count") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreInt(*value, settings.MinPartitions, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - if (!StoreInt(*value, settings.MinPartitions, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be an integer"; - return false; - } } else if (to_lower(id.Name) == "auto_partitioning_max_partitions_count") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreInt(*value, settings.MaxPartitions, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - if (!StoreInt(*value, settings.MaxPartitions, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be an integer"; - return false; - } } else if (to_lower(id.Name) == "uniform_partitions") { if (alter) { - Ctx.Error() << to_upper(id.Name) << " alter is not supported"; + Ctx.Error() << to_upper(id.Name) << " alter is not supported"; return false; } - if (!StoreInt(*value, settings.UniformPartitions, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be an integer"; + if (!StoreInt(*value, settings.UniformPartitions, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } } else if (to_lower(id.Name) == "partition_at_keys") { if (alter) { - Ctx.Error() << to_upper(id.Name) << " alter is not supported"; + Ctx.Error() << to_upper(id.Name) << " alter is not supported"; return false; } TSqlExpression expr(Ctx, Mode); - if (!StoreSplitBoundaries(*value, settings.PartitionAtKeys, expr, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be a list of keys. " + if (!StoreSplitBoundaries(*value, settings.PartitionAtKeys, expr, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a list of keys. " << "Example1: (10, 1000) Example2: ((10), (1000, \"abc\"))"; return false; } } else if (to_lower(id.Name) == "key_bloom_filter") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + return false; + } + if (!StoreId(*value, settings.KeyBloomFilter, *this)) { + Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - if (!StoreId(*value, settings.KeyBloomFilter, *this)) { - Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; - return false; - } } else if (to_lower(id.Name) == "read_replicas_settings") { - if (reset) { - Ctx.Error() << to_upper(id.Name) << " reset is not supported"; + if (reset) { + Ctx.Error() << to_upper(id.Name) << " reset is not supported"; return false; } - if (!StoreString(*value, settings.ReadReplicasSettings, Ctx)) { - Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; - return false; - } - } else if (to_lower(id.Name) == "ttl") { - if (!reset) { - TSqlExpression expr(Ctx, Mode); - if (!StoreTtlSettings(*value, settings.TtlSettings, expr, Ctx, *this)) { - Ctx.Error() << "Invalid TTL settings"; - return false; - } - } else { - settings.TtlSettings.Reset(); - } + if (!StoreString(*value, settings.ReadReplicasSettings, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; + return false; + } + } else if (to_lower(id.Name) == "ttl") { + if (!reset) { + TSqlExpression expr(Ctx, Mode); + if (!StoreTtlSettings(*value, settings.TtlSettings, expr, Ctx, *this)) { + Ctx.Error() << "Invalid TTL settings"; + return false; + } + } else { + settings.TtlSettings.Reset(); + } } else { Ctx.Error() << "Unknown table setting: " << id.Name; return false; @@ -2284,15 +2284,15 @@ bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule return true; } -bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, - TTableSettings& settings, bool alter) { - return StoreTableSettingsEntry(id, &value, settings, alter, false); -} - -bool TSqlTranslation::ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings) { - return StoreTableSettingsEntry(id, nullptr, settings, true, true); -} - +bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, + TTableSettings& settings, bool alter) { + return StoreTableSettingsEntry(id, &value, settings, alter, false); +} + +bool TSqlTranslation::ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings) { + return StoreTableSettingsEntry(id, nullptr, settings, true, true); +} + bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings) { const auto& firstEntry = settingsNode.GetRule_table_settings_entry3(); if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), @@ -8154,15 +8154,15 @@ private: bool AlterTableAlterColumn(const TRule_alter_table_alter_column& node, TAlterTableParameters& params); bool AlterTableAddFamily(const TRule_family_entry& node, TAlterTableParameters& params); bool AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, TAlterTableParameters& params); - bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params); - bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, TAlterTableParameters& params); - bool AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, TAlterTableParameters& params); + bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params); + bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, TAlterTableParameters& params); + bool AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, TAlterTableParameters& params); bool AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params); void AlterTableDropIndex(const TRule_alter_table_drop_index& node, TAlterTableParameters& params); void AlterTableRenameTo(const TRule_alter_table_rename_to& node, TAlterTableParameters& params); - bool AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params); - bool AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params); - void AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params); + bool AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params); + bool AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params); + void AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params); TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success); void AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node); @@ -8877,29 +8877,29 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab break; } case TRule_alter_table_action::kAltAlterTableAction6: { - // SET (uncompat) - const auto& setRule = node.GetAlt_alter_table_action6().GetRule_alter_table_set_table_setting_uncompat1(); + // SET (uncompat) + const auto& setRule = node.GetAlt_alter_table_action6().GetRule_alter_table_set_table_setting_uncompat1(); if (!AlterTableSetTableSetting(setRule, params)) { return false; } break; } - case TRule_alter_table_action::kAltAlterTableAction7: { - // SET (compat) - const auto& setRule = node.GetAlt_alter_table_action7().GetRule_alter_table_set_table_setting_compat1(); - if (!AlterTableSetTableSetting(setRule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction8: { - // RESET - const auto& setRule = node.GetAlt_alter_table_action8().GetRule_alter_table_reset_table_setting1(); - if (!AlterTableResetTableSetting(setRule, params)) { - return false; - } - break; - } + case TRule_alter_table_action::kAltAlterTableAction7: { + // SET (compat) + const auto& setRule = node.GetAlt_alter_table_action7().GetRule_alter_table_set_table_setting_compat1(); + if (!AlterTableSetTableSetting(setRule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction8: { + // RESET + const auto& setRule = node.GetAlt_alter_table_action8().GetRule_alter_table_reset_table_setting1(); + if (!AlterTableResetTableSetting(setRule, params)) { + return false; + } + break; + } case TRule_alter_table_action::kAltAlterTableAction9: { // ADD INDEX const auto& addIndex = node.GetAlt_alter_table_action9().GetRule_alter_table_add_index1(); @@ -8926,28 +8926,28 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab AlterTableRenameTo(renameTo, params); break; } - case TRule_alter_table_action::kAltAlterTableAction12: { - // ADD CHANGEFEED - const auto& rule = node.GetAlt_alter_table_action12().GetRule_alter_table_add_changefeed1(); - if (!AlterTableAddChangefeed(rule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction13: { - // ALTER CHANGEFEED - const auto& rule = node.GetAlt_alter_table_action13().GetRule_alter_table_alter_changefeed1(); - if (!AlterTableAlterChangefeed(rule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction14: { - // DROP CHANGEFEED - const auto& rule = node.GetAlt_alter_table_action14().GetRule_alter_table_drop_changefeed1(); - AlterTableDropChangefeed(rule, params); - break; - } + case TRule_alter_table_action::kAltAlterTableAction12: { + // ADD CHANGEFEED + const auto& rule = node.GetAlt_alter_table_action12().GetRule_alter_table_add_changefeed1(); + if (!AlterTableAddChangefeed(rule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction13: { + // ALTER CHANGEFEED + const auto& rule = node.GetAlt_alter_table_action13().GetRule_alter_table_alter_changefeed1(); + if (!AlterTableAlterChangefeed(rule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction14: { + // DROP CHANGEFEED + const auto& rule = node.GetAlt_alter_table_action14().GetRule_alter_table_drop_changefeed1(); + AlterTableDropChangefeed(rule, params); + break; + } default: AltNotImplemented("alter_table_action", node); @@ -9037,7 +9037,7 @@ bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_famil return true; } -bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, +bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params) { if (!StoreTableSettingsEntry(IdEx(node.GetRule_an_id2(), *this), node.GetRule_table_setting_value3(), @@ -9047,40 +9047,40 @@ bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_sett return true; } -bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, - TAlterTableParameters& params) -{ - const auto& firstEntry = node.GetRule_alter_table_setting_entry3(); - if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), - params.TableSettings, true)) { - return false; - } - for (auto& block : node.GetBlock4()) { - const auto& entry = block.GetRule_alter_table_setting_entry2(); - if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), - params.TableSettings, true)) { - return false; - } - } - return true; -} - -bool TSqlQuery::AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, - TAlterTableParameters& params) -{ - const auto& firstEntry = node.GetRule_an_id3(); - if (!ResetTableSettingsEntry(IdEx(firstEntry, *this), params.TableSettings)) { - return false; - } - for (auto& block : node.GetBlock4()) { - const auto& entry = block.GetRule_an_id2(); - if (!ResetTableSettingsEntry(IdEx(entry, *this), params.TableSettings)) { - return false; - } - } - return true; -} - +bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, + TAlterTableParameters& params) +{ + const auto& firstEntry = node.GetRule_alter_table_setting_entry3(); + if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), + params.TableSettings, true)) { + return false; + } + for (auto& block : node.GetBlock4()) { + const auto& entry = block.GetRule_alter_table_setting_entry2(); + if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), + params.TableSettings, true)) { + return false; + } + } + return true; +} + +bool TSqlQuery::AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, + TAlterTableParameters& params) +{ + const auto& firstEntry = node.GetRule_an_id3(); + if (!ResetTableSettingsEntry(IdEx(firstEntry, *this), params.TableSettings)) { + return false; + } + for (auto& block : node.GetBlock4()) { + const auto& entry = block.GetRule_an_id2(); + if (!ResetTableSettingsEntry(IdEx(entry, *this), params.TableSettings)) { + return false; + } + } + return true; +} + bool TSqlQuery::AlterTableAddIndex(const TRule_alter_table_add_index& node, TAlterTableParameters& params) { if (!CreateTableIndex(node.GetRule_table_index2(), *this, params.AddIndexes)) { return false; @@ -9096,41 +9096,41 @@ void TSqlQuery::AlterTableRenameTo(const TRule_alter_table_rename_to& node, TAlt params.RenameTo = IdEx(node.GetRule_an_id_table3(), *this); } -bool TSqlQuery::AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params) { - return CreateChangefeed(node.GetRule_changefeed2(), *this, params.AddChangefeeds); -} - -bool TSqlQuery::AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params) { - params.AlterChangefeeds.emplace_back(IdEx(node.GetRule_an_id3(), *this)); - - const auto& alter = node.GetRule_changefeed_alter_settings4(); - switch (alter.Alt_case()) { - case TRule_changefeed_alter_settings::kAltChangefeedAlterSettings1: { - // DISABLE - params.AlterChangefeeds.back().Disable = true; - break; - } - case TRule_changefeed_alter_settings::kAltChangefeedAlterSettings2: { - // SET - const auto& rule = alter.GetAlt_changefeed_alter_settings2().GetRule_changefeed_settings3(); - if (!ChangefeedSettings(rule, *this, params.AlterChangefeeds.back().Settings, true)) { - return false; - } - break; - } - - default: - AltNotImplemented("changefeed_alter_settings", alter); - return false; - } - - return true; -} - -void TSqlQuery::AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params) { - params.DropChangefeeds.emplace_back(IdEx(node.GetRule_an_id3(), *this)); -} - +bool TSqlQuery::AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params) { + return CreateChangefeed(node.GetRule_changefeed2(), *this, params.AddChangefeeds); +} + +bool TSqlQuery::AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params) { + params.AlterChangefeeds.emplace_back(IdEx(node.GetRule_an_id3(), *this)); + + const auto& alter = node.GetRule_changefeed_alter_settings4(); + switch (alter.Alt_case()) { + case TRule_changefeed_alter_settings::kAltChangefeedAlterSettings1: { + // DISABLE + params.AlterChangefeeds.back().Disable = true; + break; + } + case TRule_changefeed_alter_settings::kAltChangefeedAlterSettings2: { + // SET + const auto& rule = alter.GetAlt_changefeed_alter_settings2().GetRule_changefeed_settings3(); + if (!ChangefeedSettings(rule, *this, params.AlterChangefeeds.back().Settings, true)) { + return false; + } + break; + } + + default: + AltNotImplemented("changefeed_alter_settings", alter); + return false; + } + + return true; +} + +void TSqlQuery::AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params) { + params.DropChangefeeds.emplace_back(IdEx(node.GetRule_an_id3(), *this)); +} + TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) { success = false; const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 33d78cb1af1..85b6e0dfcd3 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -203,21 +203,21 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); } - Y_UNIT_TEST(ResetKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); - } - - Y_UNIT_TEST(SyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SYNC (SYNC Uint32, PRIMARY KEY (SYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT SYNC FROM SYNC").IsOk()); - } - - Y_UNIT_TEST(AsyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE ASYNC (ASYNC Uint32, PRIMARY KEY (ASYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT ASYNC FROM ASYNC").IsOk()); - } - + Y_UNIT_TEST(ResetKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); + } + + Y_UNIT_TEST(SyncKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SYNC (SYNC Uint32, PRIMARY KEY (SYNC));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT SYNC FROM SYNC").IsOk()); + } + + Y_UNIT_TEST(AsyncKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE ASYNC (ASYNC Uint32, PRIMARY KEY (ASYNC));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT ASYNC FROM ASYNC").IsOk()); + } + Y_UNIT_TEST(Jubilee) { NYql::TAstParseResult res = SqlToYql("USE plato; INSERT INTO Arcadia (r2000000) VALUES (\"2M GET!!!\");"); UNIT_ASSERT(res.Root); @@ -1392,28 +1392,28 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } - - Y_UNIT_TEST(TtlParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH ( TTL = Interval("P1D") On CreatedAt);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("expireAfter")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } + + Y_UNIT_TEST(TtlParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) + WITH ( TTL = Interval("P1D") On CreatedAt);)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("expireAfter")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } Y_UNIT_TEST(CloneForAsTableWorksWithCube) { UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); @@ -2943,18 +2943,18 @@ select FormatType($f()); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" "<main>:6:39: Error: Unknown correlation name: t\n"); } - - Y_UNIT_TEST(InvalidTtl) { - auto req = R"( - USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH ( TTL = 1 On ExpireAt ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:26: Error: Literal of Interval type is expected for TTL\n" - "<main>:4:26: Error: Invalid TTL settings\n"); - } + + Y_UNIT_TEST(InvalidTtl) { + auto req = R"( + USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) + WITH ( TTL = 1 On ExpireAt ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:26: Error: Literal of Interval type is expected for TTL\n" + "<main>:4:26: Error: Invalid TTL settings\n"); + } Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { auto req = "USE plato;\n" diff --git a/ydb/library/yql/utils/resetable_setting.h b/ydb/library/yql/utils/resetable_setting.h index febe750af97..0112105a3bf 100644 --- a/ydb/library/yql/utils/resetable_setting.h +++ b/ydb/library/yql/utils/resetable_setting.h @@ -1,67 +1,67 @@ -#pragma once - -#include "yql_panic.h" - -#include <util/generic/maybe.h> -#include <util/generic/variant.h> - -namespace NYql { - -template <typename S, typename R> -class TResetableSettingBase { -protected: - using TSet = S; - using TReset = R; - -public: - void Set(const TSet& value) { - Value.ConstructInPlace(value); - } - - void Reset(const TReset& value) { - Value.ConstructInPlace(value); - } - - bool Defined() const { - return Value.Defined(); - } - - explicit operator bool() const { - return Defined(); - } - - bool IsSet() const { - YQL_ENSURE(Defined()); - return Value->index() == 0; - } - - const TSet& GetValueSet() const { - YQL_ENSURE(IsSet()); +#pragma once + +#include "yql_panic.h" + +#include <util/generic/maybe.h> +#include <util/generic/variant.h> + +namespace NYql { + +template <typename S, typename R> +class TResetableSettingBase { +protected: + using TSet = S; + using TReset = R; + +public: + void Set(const TSet& value) { + Value.ConstructInPlace(value); + } + + void Reset(const TReset& value) { + Value.ConstructInPlace(value); + } + + bool Defined() const { + return Value.Defined(); + } + + explicit operator bool() const { + return Defined(); + } + + bool IsSet() const { + YQL_ENSURE(Defined()); + return Value->index() == 0; + } + + const TSet& GetValueSet() const { + YQL_ENSURE(IsSet()); return std::get<TSet>(*Value); - } - - const TReset& GetValueReset() const { - YQL_ENSURE(!IsSet()); + } + + const TReset& GetValueReset() const { + YQL_ENSURE(!IsSet()); return std::get<TReset>(*Value); - } - -private: + } + +private: TMaybe<std::variant<TSet, TReset>> Value; -}; - -template <typename S, typename R> -class TResetableSetting: public TResetableSettingBase<S, R> { -}; - -template <typename S> -class TResetableSetting<S, void>: public TResetableSettingBase<S, TNothing> { -private: - const TNothing& GetValueReset() const; - -public: - void Reset() { - TResetableSettingBase<S, TNothing>::Reset(Nothing()); - } -}; - -} +}; + +template <typename S, typename R> +class TResetableSetting: public TResetableSettingBase<S, R> { +}; + +template <typename S> +class TResetableSetting<S, void>: public TResetableSettingBase<S, TNothing> { +private: + const TNothing& GetValueReset() const; + +public: + void Reset() { + TResetableSettingBase<S, TNothing>::Reset(Nothing()); + } +}; + +} diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make index f02c3931eba..afba70be535 100644 --- a/ydb/library/yql/utils/ya.make +++ b/ydb/library/yql/utils/ya.make @@ -25,7 +25,7 @@ SRCS( proc_alive.h rand_guid.cpp rand_guid.h - resetable_setting.h + resetable_setting.h retry.cpp retry.h swap_bytes.cpp diff --git a/ydb/public/api/grpc/ya.make b/ydb/public/api/grpc/ya.make index 91a18c26511..24173a0264d 100644 --- a/ydb/public/api/grpc/ya.make +++ b/ydb/public/api/grpc/ya.make @@ -15,8 +15,8 @@ SRCS( ydb_auth_v1.proto ydb_coordination_v1.proto ydb_discovery_v1.proto - ydb_export_v1.proto - ydb_import_v1.proto + ydb_export_v1.proto + ydb_import_v1.proto ydb_monitoring_v1.proto ydb_operation_v1.proto ydb_cms_v1.proto diff --git a/ydb/public/api/grpc/ydb_export_v1.proto b/ydb/public/api/grpc/ydb_export_v1.proto index 1829f04eb65..e3768157751 100644 --- a/ydb/public/api/grpc/ydb_export_v1.proto +++ b/ydb/public/api/grpc/ydb_export_v1.proto @@ -1,17 +1,17 @@ -syntax = "proto3"; - -package Ydb.Export.V1; -option java_package = "com.yandex.ydb.export.v1"; - +syntax = "proto3"; + +package Ydb.Export.V1; +option java_package = "com.yandex.ydb.export.v1"; + import "ydb/public/api/protos/ydb_export.proto"; - -service ExportService { - - // Exports data to YT. - // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToYt(Export.ExportToYtRequest) returns (Export.ExportToYtResponse); - - // Exports data to S3. - // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response); -} + +service ExportService { + + // Exports data to YT. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ExportToYt(Export.ExportToYtRequest) returns (Export.ExportToYtResponse); + + // Exports data to S3. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response); +} diff --git a/ydb/public/api/grpc/ydb_import_v1.proto b/ydb/public/api/grpc/ydb_import_v1.proto index 0cd5d09fff4..192a78bd6d3 100644 --- a/ydb/public/api/grpc/ydb_import_v1.proto +++ b/ydb/public/api/grpc/ydb_import_v1.proto @@ -1,17 +1,17 @@ -syntax = "proto3"; - -package Ydb.Import.V1; -option java_package = "com.yandex.ydb.import_.v1"; - +syntax = "proto3"; + +package Ydb.Import.V1; +option java_package = "com.yandex.ydb.import_.v1"; + import "ydb/public/api/protos/ydb_import.proto"; - -service ImportService { - - // Imports data from S3. - // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); - - // Writes data to a table. - // Method accepts serialized data in the selected format and writes it non-transactionally. - rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); -} + +service ImportService { + + // Imports data from S3. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); + + // Writes data to a table. + // Method accepts serialized data in the selected format and writes it non-transactionally. + rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); +} diff --git a/ydb/public/api/grpc/ydb_operation_v1.proto b/ydb/public/api/grpc/ydb_operation_v1.proto index 1e659d2af17..8202d084843 100644 --- a/ydb/public/api/grpc/ydb_operation_v1.proto +++ b/ydb/public/api/grpc/ydb_operation_v1.proto @@ -24,16 +24,16 @@ service OperationService { // Check status for a given operation. rpc GetOperation(Operations.GetOperationRequest) returns (Operations.GetOperationResponse); - - // Starts cancellation of a long-running operation, - // Clients can use GetOperation to check whether the cancellation succeeded - // or whether the operation completed despite cancellation. - rpc CancelOperation(Operations.CancelOperationRequest) returns (Operations.CancelOperationResponse); - - // Forgets long-running operation. It does not cancel the operation and returns - // an error if operation was not completed. - rpc ForgetOperation(Operations.ForgetOperationRequest) returns (Operations.ForgetOperationResponse); - - // Lists operations that match the specified filter in the request. - rpc ListOperations(Operations.ListOperationsRequest) returns (Operations.ListOperationsResponse); + + // Starts cancellation of a long-running operation, + // Clients can use GetOperation to check whether the cancellation succeeded + // or whether the operation completed despite cancellation. + rpc CancelOperation(Operations.CancelOperationRequest) returns (Operations.CancelOperationResponse); + + // Forgets long-running operation. It does not cancel the operation and returns + // an error if operation was not completed. + rpc ForgetOperation(Operations.ForgetOperationRequest) returns (Operations.ForgetOperationResponse); + + // Lists operations that match the specified filter in the request. + rpc ListOperations(Operations.ListOperationsRequest) returns (Operations.ListOperationsResponse); } diff --git a/ydb/public/api/protos/annotations/validation.proto b/ydb/public/api/protos/annotations/validation.proto index 1c7221a3482..70d60857ed1 100644 --- a/ydb/public/api/protos/annotations/validation.proto +++ b/ydb/public/api/protos/annotations/validation.proto @@ -1,36 +1,36 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb; -option java_package = "com.yandex.ydb"; - -import "google/protobuf/descriptor.proto"; - -message Limit { - message Range { - uint32 min = 1; - uint32 max = 2; - } - - oneof kind { - Range range = 1; - uint32 lt = 2; - uint32 le = 3; - uint32 eq = 4; - uint32 ge = 5; - uint32 gt = 6; - } -} - -message MapKey { - Limit length = 1; - string value = 2; -} - -extend google.protobuf.FieldOptions { - bool required = 87650; - Limit size = 87651; - Limit length = 87652; - MapKey map_key = 87653; - string value = 87654; -} +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb; +option java_package = "com.yandex.ydb"; + +import "google/protobuf/descriptor.proto"; + +message Limit { + message Range { + uint32 min = 1; + uint32 max = 2; + } + + oneof kind { + Range range = 1; + uint32 lt = 2; + uint32 le = 3; + uint32 eq = 4; + uint32 ge = 5; + uint32 gt = 6; + } +} + +message MapKey { + Limit length = 1; + string value = 2; +} + +extend google.protobuf.FieldOptions { + bool required = 87650; + Limit size = 87651; + Limit length = 87652; + MapKey map_key = 87653; + string value = 87654; +} diff --git a/ydb/public/api/protos/annotations/ya.make b/ydb/public/api/protos/annotations/ya.make index e49e4ab62ef..a4d2dab4833 100644 --- a/ydb/public/api/protos/annotations/ya.make +++ b/ydb/public/api/protos/annotations/ya.make @@ -1,15 +1,15 @@ -PROTO_LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -SRCS( +PROTO_LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +SRCS( sensitive.proto - validation.proto -) - + validation.proto +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/api/protos/draft/datastreams.proto b/ydb/public/api/protos/draft/datastreams.proto index d29fc94db25..5c1354288af 100644 --- a/ydb/public/api/protos/draft/datastreams.proto +++ b/ydb/public/api/protos/draft/datastreams.proto @@ -3,12 +3,12 @@ option cc_enable_arenas = true; import "ydb/public/api/protos/ydb_operation.proto"; -import "google/protobuf/descriptor.proto"; +import "google/protobuf/descriptor.proto"; -package Ydb.DataStreams.V1; +package Ydb.DataStreams.V1; option java_package = "com.yandex.ydb.datastreams.v1"; -// Extsions to simplify json <-> proto conversion +// Extsions to simplify json <-> proto conversion enum EFieldTransformationType { TRANSFORM_NONE = 0; TRANSFORM_BASE64 = 1; @@ -20,7 +20,7 @@ extend google.protobuf.FieldOptions { EFieldTransformationType FieldTransformer = 58123; } -// Here and below: Kinesis data types mapped to protobuf +// Here and below: Kinesis data types mapped to protobuf enum EncryptionType { ENCRYPTION_UNDEFINED = 0; @@ -29,105 +29,105 @@ enum EncryptionType { } message EnhancedMetrics { - // List of shard-level metrics + // List of shard-level metrics repeated string shard_level_metrics = 1; } - + message ChildShard { - // The range of possible hash key values for the shard + // The range of possible hash key values for the shard HashKeyRange hash_key_range = 1; - // List of parent shards + // List of parent shards repeated string parent_shards = 2; - // Name of the shard - string name = 3; + // Name of the shard + string name = 3; } -// Represents details of consumer +// Represents details of consumer message Consumer { string consumer_arn = 1; int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; string consumer_name = 3; ConsumerDescription.ConsumerStatus consumer_status = 4; } - -// The range of possible hash key values for the shard + +// The range of possible hash key values for the shard message HashKeyRange { string starting_hash_key = 1; string ending_hash_key = 2; } - + message Record { - // Timestamp that the record was inserted into the stream + // Timestamp that the record was inserted into the stream int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Data blob + // Data blob bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; - // Encryption type used on record + // Encryption type used on record EncryptionType encryption = 3; - // Identifies shard in the stream the record is assigned to + // Identifies shard in the stream the record is assigned to string partition_key = 4; - // Unique id of the record within shard + // Unique id of the record within shard string sequence_number = 5; } - -// Represents the stream description + +// Represents the stream description message StreamDescription { - // Represents current status of the stream being described + // Represents current status of the stream being described enum StreamStatus { STATUS_UNDEFINED = 0; - // Stream is being created + // Stream is being created CREATING = 1; - // Stream is being deleted + // Stream is being deleted DELETING = 2; - // Stream is ready for read & write operations or deletion + // Stream is ready for read & write operations or deletion ACTIVE = 3; - // Shards in the stream are being merged ot split + // Shards in the stream are being merged ot split UPDATING = 4; } - // Encryption type used on record + // Encryption type used on record EncryptionType encryption_type = 1; - // List of enhanced monitoring settings + // List of enhanced monitoring settings repeated EnhancedMetrics enhanced_monitoring = 2; - // If true, more shards are available to describe + // If true, more shards are available to describe bool has_more_shards = 3; - // Id of the key used to encryption - string key_id = 4; - // Retention period in hours + // Id of the key used to encryption + string key_id = 4; + // Retention period in hours int32 retention_period_hours = 5; - // List of shards + // List of shards repeated Shard shards = 6; string stream_arn = 7; - // Timestamp that the stream was created + // Timestamp that the stream was created int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Current status of the stream + // Current status of the stream StreamStatus stream_status = 9; - // Name of the stream + // Name of the stream string stream_name = 10; int32 write_quota_kb_per_sec = 11; string owner = 12; } -// Represents range of possible sequence numbers for the shard +// Represents range of possible sequence numbers for the shard message SequenceNumberRange { string starting_sequence_number = 1; string ending_sequence_number = 2; } -// Represents shard details +// Represents shard details message Shard { - // Id of the shard adjacent to the shard's parent + // Id of the shard adjacent to the shard's parent string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; - // The range of possible hash key values for the shard + // The range of possible hash key values for the shard HashKeyRange hash_key_range = 2; - // Id of the shard's parent + // Id of the shard's parent string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; - // The range of possible sequence numbers for the shard + // The range of possible sequence numbers for the shard SequenceNumberRange sequence_number_range = 4; - // Unique id of the shard within stream + // Unique id of the shard within stream string shard_id = 5; } -// Represents consumer details +// Represents consumer details message ConsumerDescription { enum ConsumerStatus { STATUS_UNKNOWN = 0; @@ -137,98 +137,98 @@ message ConsumerDescription { } string consumer_arn = 1; - // Timestamp that the consumer was created + // Timestamp that the consumer was created int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Name of the consumer + // Name of the consumer string consumer_name = 3; ConsumerStatus consumer_status = 4; string stream_arn = 5; } -// Determines how the shard iterator is used to start reading records +// Determines how the shard iterator is used to start reading records enum ShardIteratorType { SHARD_ITERATOR_UNDEFINED = 0; - // Start reading exactly from the position denoted by a sequence number + // Start reading exactly from the position denoted by a sequence number AT_SEQUENCE_NUMBER = 1; - // Start reading right after the specified position + // Start reading right after the specified position AFTER_SEQUENCE_NUMBER = 2; - // Start reading from the position denoted by a specific timestamp + // Start reading from the position denoted by a specific timestamp AT_TIMESTAMP = 3; - // Start reading from the oldest record + // Start reading from the oldest record TRIM_HORIZON = 4; - // Start reading latest records + // Start reading latest records LATEST = 5; } -// Parameters used to filter out the response of ListShards +// Parameters used to filter out the response of ListShards message ShardFilter { enum ShardFilterType { SHARD_TYPE_UNDEFINED = 0; - // Include all the shards, starting with the shard whose id immediately follows the shard_id + // Include all the shards, starting with the shard whose id immediately follows the shard_id AFTER_SHARD_ID = 1; - // Include all the shards that were open at TRIM_HORIZON + // Include all the shards that were open at TRIM_HORIZON AT_TRIM_HORIZON = 2; - // Include all the shards within the retention period + // Include all the shards within the retention period FROM_TRIM_HORIZON = 3; - // Include only the currently open shards + // Include only the currently open shards AT_LATEST = 4; // Include all the shards whose start timestamp is <= to the given AT_TIMESTAMP = 5; - // Include all closed shards whose end timestamp >= to the given + // Include all closed shards whose end timestamp >= to the given FROM_TIMESTAMP = 6; } - // Exclusive id. Can only be used if AFTER_SHARD_ID is specified + // Exclusive id. Can only be used if AFTER_SHARD_ID is specified string shard_id = 1; - // Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified. + // Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified. int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; ShardFilterType type = 3; } - -// Represents starting position in the stream from which to start reading + +// Represents starting position in the stream from which to start reading message StartingPosition { - // Timestamp of the record from which to start reading + // Timestamp of the record from which to start reading int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Sequence number of the record from which to start reading + // Sequence number of the record from which to start reading string sequence_number = 2; ShardIteratorType type = 3; } message StreamDescriptionSummary { - // Number of enhanced fan-out consumers + // Number of enhanced fan-out consumers int32 consumer_count = 1; - // Encryption type used + // Encryption type used EncryptionType encryption_type = 2; - // List of enhanced monitoring settings + // List of enhanced monitoring settings repeated EnhancedMetrics enhanced_monitoring = 3; - // Id of the key used to encryption + // Id of the key used to encryption string key_id = 4; - // Number of open shards in the stream + // Number of open shards in the stream int32 open_shard_count = 5; - // Retention period in hours + // Retention period in hours int32 retention_period_hours = 6; string stream_arn = 7; - // Timestamp that the stream was created + // Timestamp that the stream was created int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Name of the stream + // Name of the stream string stream_name = 9; - // Current status of the stream + // Current status of the stream StreamDescription.StreamStatus stream_status = 10; } - -// Server sends events of this type after call SubscribeToShard + +// Server sends events of this type after call SubscribeToShard message SubscribeToShardEvent { - // List of child shards + // List of child shards repeated ChildShard child_shards = 1; - // Used in next call to SubscribeToShards + // Used in next call to SubscribeToShards string continuation_sequence_number = 2; - // Number of milliseconds the read records are from the tip of the stream. - // Zero indicates that record processing is caught up. + // Number of milliseconds the read records are from the tip of the stream. + // Zero indicates that record processing is caught up. int32 millis_behind_latest = 3; - // List of records + // List of records repeated Record records = 4; } - + message SubscribeToShardEventStream { message ExceptionEvent { } @@ -246,8 +246,8 @@ message SubscribeToShardEventStream { SubscribeToShardEvent subscribe_to_shard_event = 10; } } - -// Represents metadata assigned to the stream (key-value pair) + +// Represents metadata assigned to the stream (key-value pair) message Tag { string key = 1; string value = 2; @@ -255,154 +255,154 @@ message Tag { message CreateStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; - // Shards count for a new stream + // Shards count for a new stream int32 shard_count = 3; - // Datastreams extension. Allow to set retention and write quota upon stream creation + // Datastreams extension. Allow to set retention and write quota upon stream creation int32 write_quota_kb_per_sec = 4; int32 retention_period_hours = 5; -} +} message CreateStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message CreateStreamResult { } message DeleteStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // If unset or false, and the stream has registered consumers, the call to DeleteStream fails + // If unset or false, and the stream has registered consumers, the call to DeleteStream fails bool enforce_consumer_deletion = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } - + message DeleteStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DeleteStreamResult { } message ListStreamsRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of stream to start from - string exclusive_start_stream_name = 2; - // Max number of entries to return - int32 limit = 3; + // Name of stream to start from + string exclusive_start_stream_name = 2; + // Max number of entries to return + int32 limit = 3; // Will make recurse listing if set to true. Otherwise will return only streams from root directory. bool recurse = 4; } - + message ListStreamsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message ListStreamsResult { - // True if more streams are available - bool has_more_streams = 1; - // Name of streams - repeated string stream_names = 2; + // True if more streams are available + bool has_more_streams = 1; + // Name of streams + repeated string stream_names = 2; } message ListShardsRequest { Ydb.Operations.OperationParams operation_params = 1; - // Id of the shard to start from + // Id of the shard to start from string exclusive_start_shard_id = 2; - // Max number of entries to return + // Max number of entries to return int32 max_results = 3; - // Pagination token - string next_token = 4; - // Filter out response - ShardFilter shard_filter = 5; - // Used to distinguish streams that have the same name + // Pagination token + string next_token = 4; + // Filter out response + ShardFilter shard_filter = 5; + // Used to distinguish streams that have the same name int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; - // Name of the stream + // Name of the stream string stream_name = 7; } - + message ListShardsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message ListShardsResult { - // Pagination token + // Pagination token string next_token = 1; - // List of shards + // List of shards repeated Shard shards = 2; } message SetWriteQuotaRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; int32 write_quota_kb_per_sec = 3; } - + message SetWriteQuotaResponse { - // Result of request will be inside operation. + // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message SetWriteQuotaResult { } message UpdateStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; int32 write_quota_kb_per_sec = 3; - // New number of shards + // New number of shards int32 target_shard_count = 4; - // Retention period in hours + // Retention period in hours int32 retention_period_hours = 5; } - + message UpdateStreamResponse { - // Result of request will be inside operation. + // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message UpdateStreamResult { } message ListStreamConsumersRequest { Ydb.Operations.OperationParams operation_params = 1; - // Max number of entries to return + // Max number of entries to return int32 max_results = 2; - // Pagination token + // Pagination token string next_token = 3; string stream_arn = 4; - // Used to distinguish streams that have the same name + // Used to distinguish streams that have the same name int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; } - + message ListStreamConsumersResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message ListStreamConsumersResult { - // Pagination token + // Pagination token string next_token = 1; - // List of consumers + // List of consumers repeated Consumer consumers = 2; } message DescribeStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Id of the shard to start from + // Id of the shard to start from string exclusive_start_shard_id = 2; - // Max number of entries to return + // Max number of entries to return int32 limit = 3; - // Name of the stream to describe + // Name of the stream to describe string stream_name = 4; } @@ -410,23 +410,23 @@ message DescribeStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DescribeStreamResult { StreamDescription stream_description = 1; } message RegisterStreamConsumerRequest { Ydb.Operations.OperationParams operation_params = 1; - // Unique name of the consumer within the stream + // Unique name of the consumer within the stream string consumer_name = 2; string stream_arn = 3; } - + message RegisterStreamConsumerResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message RegisterStreamConsumerResult { Consumer consumer = 1; } @@ -434,46 +434,46 @@ message RegisterStreamConsumerResult { message DeregisterStreamConsumerRequest { Ydb.Operations.OperationParams operation_params = 1; string consumer_arn = 2; - // Name of consumer + // Name of consumer string consumer_name = 3; string stream_arn = 4; } - + message DeregisterStreamConsumerResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DeregisterStreamConsumerResult { } message DescribeStreamConsumerRequest { Ydb.Operations.OperationParams operation_params = 1; string consumer_arn = 2; - // Name of consumer + // Name of consumer string consumer_name = 3; string stream_arn = 4; } - + message DescribeStreamConsumerResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DescribeStreamConsumerResult { ConsumerDescription consumer_description = 1; } message PutRecordsRequestEntry { - // Data blob + // Data blob bytes data = 1 [(FieldTransformer) = TRANSFORM_BASE64]; - // Hash value used to explicitly determine the shard + // Hash value used to explicitly determine the shard string explicit_hash_key = 2; - // Used as input to hash function that maps partition key to a specific shard + // Used as input to hash function that maps partition key to a specific shard string partition_key = 3; } -// Represents result of an individual record +// Represents result of an individual record message PutRecordsResultEntry { string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; @@ -483,95 +483,95 @@ message PutRecordsResultEntry { message GetRecordsRequest { Ydb.Operations.OperationParams operation_params = 1; - // Max number of records to return + // Max number of records to return int32 limit = 2; - // Iterator to shard - string shard_iterator = 3; -} + // Iterator to shard + string shard_iterator = 3; +} message GetRecordsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message GetRecordsResult { - // List of child shards if SplitShard was called on a given shard - repeated ChildShard child_shards = 1; - // Number of milliseconds the read records are from the tip of the stream. - // Zero indicates that record processing is caught up. + // List of child shards if SplitShard was called on a given shard + repeated ChildShard child_shards = 1; + // Number of milliseconds the read records are from the tip of the stream. + // Zero indicates that record processing is caught up. int64 millis_behind_latest = 2; string next_shard_iterator = 3; - // List of records - repeated Record records = 4; + // List of records + repeated Record records = 4; } message PutRecordRequest { Ydb.Operations.OperationParams operation_params = 1; - // Data blob - bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; - // Hash value used to explicitly determine the shard - string explicit_hash_key = 3; - // Used as input to hash function that maps partition key to a specific shard + // Data blob + bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; + // Hash value used to explicitly determine the shard + string explicit_hash_key = 3; + // Used as input to hash function that maps partition key to a specific shard string partition_key = 4; string sequence_number_for_ordering = 5; - // Name of the stream to put record to - string stream_name = 6; + // Name of the stream to put record to + string stream_name = 6; } - + message PutRecordResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message PutRecordResult { - // Encryption type used + // Encryption type used EncryptionType encryption_type = 1; - // Unique id of the record within shard + // Unique id of the record within shard string sequence_number = 2; - // Id of the shard were the data was inserted + // Id of the shard were the data was inserted string shard_id = 3; } message PutRecordsRequest { Ydb.Operations.OperationParams operation_params = 1; - // List of records + // List of records repeated PutRecordsRequestEntry records = 2; - // Name of the stream to put records to + // Name of the stream to put records to string stream_name = 3; } - + message PutRecordsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message PutRecordsResult { - // Encryption type used + // Encryption type used EncryptionType encryption_type = 1; - // Number of unsuccessfully processed records + // Number of unsuccessfully processed records int32 failed_record_count = 2; - // List of results + // List of results repeated PutRecordsResultEntry records = 3; } message GetShardIteratorRequest { Ydb.Operations.OperationParams operation_params = 1; - // Id of the shard to get iterator for + // Id of the shard to get iterator for string shard_id = 2; ShardIteratorType shard_iterator_type = 3; - // Used with shard iterator type AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER + // Used with shard iterator type AT_SEQUENCE_NUMBER and AFTER_SEQUENCE_NUMBER string starting_sequence_number = 4; - // Name of the stream + // Name of the stream string stream_name = 5; - // Used with shard iterator type AT_TIMESTAMP + // Used with shard iterator type AT_TIMESTAMP int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; } - + message GetShardIteratorResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message GetShardIteratorResult { string shard_iterator = 1; } @@ -579,79 +579,79 @@ message GetShardIteratorResult { message SubscribeToShardRequest { Ydb.Operations.OperationParams operation_params = 1; string consumer_arn = 2; - // Id of the shard to subscribe to + // Id of the shard to subscribe to string shard_id = 3; StartingPosition starting_position = 4; } - + message SubscribeToShardResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message SubscribeToShardResult { SubscribeToShardEventStream event_stream = 1; } message AddTagsToStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; - // List of tags to add to stream + // List of tags to add to stream repeated string tags = 3; } - + message AddTagsToStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message AddTagsToStreamResult { } message DecreaseStreamRetentionPeriodRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; // Retention period in hours int32 retention_period_hours = 3; } - + message DecreaseStreamRetentionPeriodResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DecreaseStreamRetentionPeriodResult { } message DescribeLimitsRequest { Ydb.Operations.OperationParams operation_params = 1; } - + message DescribeLimitsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DescribeLimitsResult { - // Number of open shards + // Number of open shards int32 open_shard_count = 1; - // Max number of shards + // Max number of shards int32 shard_limit = 2; } message DescribeStreamSummaryRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; } - + message DescribeStreamSummaryResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DescribeStreamSummaryResult { // Stream description sumary StreamDescriptionSummary stream_description_summary = 1; @@ -659,200 +659,200 @@ message DescribeStreamSummaryResult { message DisableEnhancedMonitoringRequest { Ydb.Operations.OperationParams operation_params = 1; - // List of shard-level metrics to disable + // List of shard-level metrics to disable repeated string shard_level_metrics = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } - + message DisableEnhancedMonitoringResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message DisableEnhancedMonitoringResult { - // Represents current state before the operation + // Represents current state before the operation repeated string current_shard_level_metrics = 1; - // Represents list of metrics after the operation + // Represents list of metrics after the operation repeated string desired_shard_level_metrics = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } message EnableEnhancedMonitoringRequest { Ydb.Operations.OperationParams operation_params = 1; - // List of shard-level metrics to enable + // List of shard-level metrics to enable repeated string shard_level_metrics = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } - + message EnableEnhancedMonitoringResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message EnableEnhancedMonitoringResult { - // Represents current state before the operation + // Represents current state before the operation repeated string current_shard_level_metrics = 1; - // Represents list of metrics after the operation + // Represents list of metrics after the operation repeated string desired_shard_level_metrics = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } message IncreaseStreamRetentionPeriodRequest { Ydb.Operations.OperationParams operation_params = 1; - // Retention period in hours + // Retention period in hours int32 retention_period_hours = 2; - // Name of the stream + // Name of the stream string stream_name = 3; } - + message IncreaseStreamRetentionPeriodResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message IncreaseStreamRetentionPeriodResult { } message ListTagsForStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Key to start from + // Key to start from string exclusive_start_tag_key = 2; - // Max number of entries to return + // Max number of entries to return int32 limit = 3; - // Name of the stream + // Name of the stream string stream_name = 4; } - + message ListTagsForStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message ListTagsForStreamResult { - // If true, more tags are available + // If true, more tags are available bool has_more_tags = 1; - // List of tags + // List of tags repeated Tag tags = 2; } message MergeShardsRequest { Ydb.Operations.OperationParams operation_params = 1; - // Id of the adjacent shard for the merge + // Id of the adjacent shard for the merge string adjacent_shard_to_merge = 2; - // Id of the shard to combine with the adjacent + // Id of the shard to combine with the adjacent string shard_to_merge = 3; - // Name of the stream + // Name of the stream string stream_name = 4; } - + message MergeShardsResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message MergeShardsResult { } message RemoveTagsFromStreamRequest { Ydb.Operations.OperationParams operation_params = 1; - // Name of the stream + // Name of the stream string stream_name = 2; - // List of tag keys + // List of tag keys repeated string tag_keys = 3; } - + message RemoveTagsFromStreamResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message RemoveTagsFromStreamResult { } message SplitShardRequest { Ydb.Operations.OperationParams operation_params = 1; - // Hash value for the starting hash key of one of the new child shards + // Hash value for the starting hash key of one of the new child shards string new_starting_hash_key = 2; - // Id of the shard to split + // Id of the shard to split string shard_to_split = 3; - // Name of the stream + // Name of the stream string stream_name = 4; } - + message SplitShardResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message SplitShardResult { } message StartStreamEncryptionRequest { Ydb.Operations.OperationParams operation_params = 1; - // Encryption type to use + // Encryption type to use EncryptionType encryption_type = 2; - // Id of the key to use to encryption + // Id of the key to use to encryption string key_id = 3; - // Name of the stream + // Name of the stream string stream_name = 4; } - + message StartStreamEncryptionResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message StartStreamEncryptionResult { } message StopStreamEncryptionRequest { Ydb.Operations.OperationParams operation_params = 1; - // Encryption type + // Encryption type EncryptionType encryption_type = 2; - // Id of the key to use to encryption + // Id of the key to use to encryption string key_id = 3; - // Name of the stream + // Name of the stream string stream_name = 4; } - + message StopStreamEncryptionResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message StopStreamEncryptionResult { } message UpdateShardCountRequest { enum ScalingType { SCALING_TYPE_UNKNOWN = 0; - // Creates shards of equal size + // Creates shards of equal size UNIFORM_SCALING = 1; } Ydb.Operations.OperationParams operation_params = 1; - // Scaling type + // Scaling type ScalingType scaling_type = 2; - // Name of the stream + // Name of the stream string stream_name = 3; - // New number of shards + // New number of shards int32 target_shard_count = 4; } - + message UpdateShardCountResponse { // Result of request will be inside operation. Ydb.Operations.Operation operation = 1; } - + message UpdateShardCountResult { - // Current number of shards + // Current number of shards int32 current_shard_count = 1; - // Name of the stream + // Name of the stream string stream_name = 2; - // Updated number of shards + // Updated number of shards int32 target_shard_count = 3; } diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index 92387916694..8a156403a3d 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -9,10 +9,10 @@ OWNER( g:kikimr ) -PEERDIR( +PEERDIR( ydb/public/api/protos/annotations -) - +) + SRCS( draft/datastreams.proto draft/persqueue_common.proto @@ -30,9 +30,9 @@ SRCS( ydb_coordination.proto ydb_discovery.proto ydb_experimental.proto - ydb_export.proto + ydb_export.proto ydb_formats.proto - ydb_import.proto + ydb_import.proto ydb_issue_message.proto ydb_monitoring.proto ydb_operation.proto diff --git a/ydb/public/api/protos/ydb_cms.proto b/ydb/public/api/protos/ydb_cms.proto index cdd4e68aba3..1a569c70812 100644 --- a/ydb/public/api/protos/ydb_cms.proto +++ b/ydb/public/api/protos/ydb_cms.proto @@ -48,11 +48,11 @@ message Resources { repeated ComputationalUnits computational_units = 2; } -message ServerlessResources { - // Full path to shared database's home dir whose resources will be used. - string shared_database_path = 1; -} - +message ServerlessResources { + // Full path to shared database's home dir whose resources will be used. + string shared_database_path = 1; +} + message DatabaseOptions { // Do not initialize services required for transactions processing. bool disable_tx_service = 1; @@ -91,9 +91,9 @@ message DatabaseQuotas { uint64 data_stream_shards_quota = 3; // A maximum storage that will be reserved for all data stream shards. uint64 data_stream_reserved_storage_quota = 5; - // A minimum value of `TtlSettings.run_interval_seconds` that can be specified. - // Default is 1800 (15 minutes). - uint32 ttl_min_run_internal_seconds = 4; + // A minimum value of `TtlSettings.run_interval_seconds` that can be specified. + // Default is 1800 (15 minutes). + uint32 ttl_min_run_internal_seconds = 4; } // Request to create a new database. For successfull creation @@ -103,14 +103,14 @@ message CreateDatabaseRequest { Ydb.Operations.OperationParams operation_params = 1; // Required. Full path to database's home dir. Used as database ID. string path = 2; - oneof resources_kind { - // Resources to allocate for database by CMS. - Resources resources = 3; - // Shared resources can be used by serverless databases. - Resources shared_resources = 6; - // If specified, the created database will be "serverless". - ServerlessResources serverless_resources = 7; - } + oneof resources_kind { + // Resources to allocate for database by CMS. + Resources resources = 3; + // Shared resources can be used by serverless databases. + Resources shared_resources = 6; + // If specified, the created database will be "serverless". + ServerlessResources serverless_resources = 7; + } // Additional database options. DatabaseOptions options = 4; // Attach attributes to database. @@ -154,12 +154,12 @@ message GetDatabaseStatusResult { string path = 1; // Current database state. State state = 2; - oneof resources_kind { - // Database resources requested for allocation. - Resources required_resources = 3; - Resources required_shared_resources = 7; - ServerlessResources serverless_resources = 8; - } + oneof resources_kind { + // Database resources requested for allocation. + Resources required_resources = 3; + Resources required_shared_resources = 7; + ServerlessResources serverless_resources = 8; + } // Database resources allocated by CMS. Resources allocated_resources = 4; // Externally allocated database resources registered in CMS. diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 30c7f61315d..31f49667a9b 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -1,129 +1,129 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/annotations/validation.proto"; import "ydb/public/api/protos/ydb_operation.proto"; - -import "google/protobuf/timestamp.proto"; - -package Ydb.Export; -option java_package = "com.yandex.ydb.export"; - -/// Common -message ExportProgress { - enum Progress { - PROGRESS_UNSPECIFIED = 0; - PROGRESS_PREPARING = 1; - PROGRESS_TRANSFER_DATA = 2; - PROGRESS_DONE = 3; - PROGRESS_CANCELLATION = 4; - PROGRESS_CANCELLED = 5; - } -} - -message ExportItemProgress { - uint32 parts_total = 1; - uint32 parts_completed = 2; - google.protobuf.Timestamp start_time = 3; - google.protobuf.Timestamp end_time = 4; -} - -/// YT -message ExportToYtSettings { - message Item { - // Database path to a table to be exported - string source_path = 1 [(required) = true]; - string destination_path = 2 [(required) = true]; - } - - string host = 1 [(required) = true]; - uint32 port = 2; - string token = 3 [(required) = true]; - repeated Item items = 4 [(size).ge = 1]; - string description = 5 [(length).le = 128]; - uint32 number_of_retries = 6; - bool use_type_v3 = 7; -} - -message ExportToYtResult { -} - -message ExportToYtMetadata { - ExportToYtSettings settings = 1; - ExportProgress.Progress progress = 2; - repeated ExportItemProgress items_progress = 3; -} - -message ExportToYtRequest { - Ydb.Operations.OperationParams operation_params = 1; - ExportToYtSettings settings = 2 [(required) = true]; -} - -message ExportToYtResponse { - // operation.result = ExportToYtResult - // operation.metadata = ExportToYtMetadata - Ydb.Operations.Operation operation = 1; -} - -/// S3 -message ExportToS3Settings { - enum Scheme { - UNSPECIFIED = 0; - HTTP = 1; - HTTPS = 2; - } - - enum StorageClass { - STORAGE_CLASS_UNSPECIFIED = 0; - STANDARD = 1; - REDUCED_REDUNDANCY = 2; - STANDARD_IA = 3; - ONEZONE_IA = 4; - INTELLIGENT_TIERING = 5; - GLACIER = 6; - DEEP_ARCHIVE = 7; - OUTPOSTS = 8; - }; - - message Item { - // Database path to a table to be exported - string source_path = 1 [(required) = true]; - - /* Tables are exported to one or more S3 objects. - The object name begins with 'destination_prefix'. - This prefix will be followed by '/data_PartNumber', where 'PartNumber' - represents the index of the part, starting at zero. - */ - string destination_prefix = 2 [(required) = true]; - } - - string endpoint = 1 [(required) = true]; - Scheme scheme = 2; // HTTPS if not specified - string bucket = 3 [(required) = true]; - string access_key = 4 [(required) = true]; - string secret_key = 5 [(required) = true]; - repeated Item items = 6 [(size).ge = 1]; - string description = 7 [(length).le = 128]; - uint32 number_of_retries = 8; - StorageClass storage_class = 9; -} - -message ExportToS3Result { -} - -message ExportToS3Metadata { - ExportToS3Settings settings = 1; - ExportProgress.Progress progress = 2; - repeated ExportItemProgress items_progress = 3; -} - -message ExportToS3Request { - Ydb.Operations.OperationParams operation_params = 1; - ExportToS3Settings settings = 2 [(required) = true]; -} - -message ExportToS3Response { - // operation.result = ExportToS3Result - // operation.metadata = ExportToS3Metadata - Ydb.Operations.Operation operation = 1; -} + +import "google/protobuf/timestamp.proto"; + +package Ydb.Export; +option java_package = "com.yandex.ydb.export"; + +/// Common +message ExportProgress { + enum Progress { + PROGRESS_UNSPECIFIED = 0; + PROGRESS_PREPARING = 1; + PROGRESS_TRANSFER_DATA = 2; + PROGRESS_DONE = 3; + PROGRESS_CANCELLATION = 4; + PROGRESS_CANCELLED = 5; + } +} + +message ExportItemProgress { + uint32 parts_total = 1; + uint32 parts_completed = 2; + google.protobuf.Timestamp start_time = 3; + google.protobuf.Timestamp end_time = 4; +} + +/// YT +message ExportToYtSettings { + message Item { + // Database path to a table to be exported + string source_path = 1 [(required) = true]; + string destination_path = 2 [(required) = true]; + } + + string host = 1 [(required) = true]; + uint32 port = 2; + string token = 3 [(required) = true]; + repeated Item items = 4 [(size).ge = 1]; + string description = 5 [(length).le = 128]; + uint32 number_of_retries = 6; + bool use_type_v3 = 7; +} + +message ExportToYtResult { +} + +message ExportToYtMetadata { + ExportToYtSettings settings = 1; + ExportProgress.Progress progress = 2; + repeated ExportItemProgress items_progress = 3; +} + +message ExportToYtRequest { + Ydb.Operations.OperationParams operation_params = 1; + ExportToYtSettings settings = 2 [(required) = true]; +} + +message ExportToYtResponse { + // operation.result = ExportToYtResult + // operation.metadata = ExportToYtMetadata + Ydb.Operations.Operation operation = 1; +} + +/// S3 +message ExportToS3Settings { + enum Scheme { + UNSPECIFIED = 0; + HTTP = 1; + HTTPS = 2; + } + + enum StorageClass { + STORAGE_CLASS_UNSPECIFIED = 0; + STANDARD = 1; + REDUCED_REDUNDANCY = 2; + STANDARD_IA = 3; + ONEZONE_IA = 4; + INTELLIGENT_TIERING = 5; + GLACIER = 6; + DEEP_ARCHIVE = 7; + OUTPOSTS = 8; + }; + + message Item { + // Database path to a table to be exported + string source_path = 1 [(required) = true]; + + /* Tables are exported to one or more S3 objects. + The object name begins with 'destination_prefix'. + This prefix will be followed by '/data_PartNumber', where 'PartNumber' + represents the index of the part, starting at zero. + */ + string destination_prefix = 2 [(required) = true]; + } + + string endpoint = 1 [(required) = true]; + Scheme scheme = 2; // HTTPS if not specified + string bucket = 3 [(required) = true]; + string access_key = 4 [(required) = true]; + string secret_key = 5 [(required) = true]; + repeated Item items = 6 [(size).ge = 1]; + string description = 7 [(length).le = 128]; + uint32 number_of_retries = 8; + StorageClass storage_class = 9; +} + +message ExportToS3Result { +} + +message ExportToS3Metadata { + ExportToS3Settings settings = 1; + ExportProgress.Progress progress = 2; + repeated ExportItemProgress items_progress = 3; +} + +message ExportToS3Request { + Ydb.Operations.OperationParams operation_params = 1; + ExportToS3Settings settings = 2 [(required) = true]; +} + +message ExportToS3Response { + // operation.result = ExportToS3Result + // operation.metadata = ExportToS3Metadata + Ydb.Operations.Operation operation = 1; +} diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 5295aa6fca2..924b49eea2d 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -1,110 +1,110 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + import "ydb/public/api/protos/annotations/validation.proto"; import "ydb/public/api/protos/ydb_operation.proto"; - -import "google/protobuf/timestamp.proto"; - -package Ydb.Import; -option java_package = "com.yandex.ydb.import_"; - -/// Common -message ImportProgress { - enum Progress { - PROGRESS_UNSPECIFIED = 0; - PROGRESS_PREPARING = 1; - PROGRESS_TRANSFER_DATA = 2; - PROGRESS_BUILD_INDEXES = 3; - PROGRESS_DONE = 4; - PROGRESS_CANCELLATION = 5; - PROGRESS_CANCELLED = 6; - } -} - -message ImportItemProgress { - uint32 parts_total = 1; - uint32 parts_completed = 2; - google.protobuf.Timestamp start_time = 3; - google.protobuf.Timestamp end_time = 4; -} - -/// S3 -message ImportFromS3Settings { - enum Scheme { - UNSPECIFIED = 0; - HTTP = 1; - HTTPS = 2; - } - - message Item { - /* YDB tables in S3 are stored in one or more objects (see ydb_export.proto). - The object name begins with 'source_prefix'. - This prefix is followed by: - * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; - * '/scheme.pb' - object with information about scheme, indexes, etc. - */ - string source_prefix = 1 [(required) = true]; - - // Database path to a table to import to. - string destination_path = 2 [(required) = true]; - } - - string endpoint = 1 [(required) = true]; - Scheme scheme = 2; // HTTPS if not specified - string bucket = 3 [(required) = true]; - string access_key = 4 [(required) = true]; - string secret_key = 5 [(required) = true]; - repeated Item items = 6 [(size).ge = 1]; - string description = 7 [(length).le = 128]; - uint32 number_of_retries = 8; -} - -message ImportFromS3Result { -} - -message ImportFromS3Metadata { - ImportFromS3Settings settings = 1; - ImportProgress.Progress progress = 2; - repeated ImportItemProgress items_progress = 3; -} - -message ImportFromS3Request { - Ydb.Operations.OperationParams operation_params = 1; - ImportFromS3Settings settings = 2 [(required) = true]; -} - -message ImportFromS3Response { - // operation.result = ImportFromS3Result - // operation.metadata = ImportFromS3Metadata - Ydb.Operations.Operation operation = 1; -} - -/// Data -message YdbDumpFormat { - repeated string columns = 1; -} - -message ImportDataResult { -} - -message ImportDataRequest { - Ydb.Operations.OperationParams operation_params = 1; - // Full path to table - string path = 2; - // Data serialized in the selected format. Restrictions: - // - sorted by primary key; - // - all keys must be from the same partition; - // - table has no global secondary indexes; - // - size of serialized data is limited to 8 MB. - bytes data = 3 [(length).le = 8388608]; - oneof format { - // Result of `ydb tools dump` - YdbDumpFormat ydb_dump = 4; - } -} - -message ImportDataResponse { - // operation.result = ImportDataResult - Ydb.Operations.Operation operation = 1; -} + +import "google/protobuf/timestamp.proto"; + +package Ydb.Import; +option java_package = "com.yandex.ydb.import_"; + +/// Common +message ImportProgress { + enum Progress { + PROGRESS_UNSPECIFIED = 0; + PROGRESS_PREPARING = 1; + PROGRESS_TRANSFER_DATA = 2; + PROGRESS_BUILD_INDEXES = 3; + PROGRESS_DONE = 4; + PROGRESS_CANCELLATION = 5; + PROGRESS_CANCELLED = 6; + } +} + +message ImportItemProgress { + uint32 parts_total = 1; + uint32 parts_completed = 2; + google.protobuf.Timestamp start_time = 3; + google.protobuf.Timestamp end_time = 4; +} + +/// S3 +message ImportFromS3Settings { + enum Scheme { + UNSPECIFIED = 0; + HTTP = 1; + HTTPS = 2; + } + + message Item { + /* YDB tables in S3 are stored in one or more objects (see ydb_export.proto). + The object name begins with 'source_prefix'. + This prefix is followed by: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc. + */ + string source_prefix = 1 [(required) = true]; + + // Database path to a table to import to. + string destination_path = 2 [(required) = true]; + } + + string endpoint = 1 [(required) = true]; + Scheme scheme = 2; // HTTPS if not specified + string bucket = 3 [(required) = true]; + string access_key = 4 [(required) = true]; + string secret_key = 5 [(required) = true]; + repeated Item items = 6 [(size).ge = 1]; + string description = 7 [(length).le = 128]; + uint32 number_of_retries = 8; +} + +message ImportFromS3Result { +} + +message ImportFromS3Metadata { + ImportFromS3Settings settings = 1; + ImportProgress.Progress progress = 2; + repeated ImportItemProgress items_progress = 3; +} + +message ImportFromS3Request { + Ydb.Operations.OperationParams operation_params = 1; + ImportFromS3Settings settings = 2 [(required) = true]; +} + +message ImportFromS3Response { + // operation.result = ImportFromS3Result + // operation.metadata = ImportFromS3Metadata + Ydb.Operations.Operation operation = 1; +} + +/// Data +message YdbDumpFormat { + repeated string columns = 1; +} + +message ImportDataResult { +} + +message ImportDataRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Full path to table + string path = 2; + // Data serialized in the selected format. Restrictions: + // - sorted by primary key; + // - all keys must be from the same partition; + // - table has no global secondary indexes; + // - size of serialized data is limited to 8 MB. + bytes data = 3 [(length).le = 8388608]; + oneof format { + // Result of `ydb tools dump` + YdbDumpFormat ydb_dump = 4; + } +} + +message ImportDataResponse { + // operation.result = ImportDataResult + Ydb.Operations.Operation operation = 1; +} diff --git a/ydb/public/api/protos/ydb_operation.proto b/ydb/public/api/protos/ydb_operation.proto index ecf7a2752ff..30ddd002a2e 100644 --- a/ydb/public/api/protos/ydb_operation.proto +++ b/ydb/public/api/protos/ydb_operation.proto @@ -21,8 +21,8 @@ message OperationParams { // Server will only reply once operation is finished (ready=true), and operation object won't be // accessible after the reply. This is a basic request-response mode. SYNC = 1; - - ASYNC = 2; + + ASYNC = 2; } OperationMode operation_mode = 1; @@ -41,9 +41,9 @@ message OperationParams { // sent back to client if it was waiting for the operation result. // In case when cancellation isn't possible, no action will be performed. google.protobuf.Duration cancel_after = 3; - - // User-defined labels of operation. - map<string, string> labels = 4 [(map_key).length.le = 128, (length).le = 128]; + + // User-defined labels of operation. + map<string, string> labels = 4 [(map_key).length.le = 128, (length).le = 128]; // If enabled, server will report cost information, if supported by the operation. // This flag is mostly useful for SYNC operations, to get the cost information in the response. @@ -51,44 +51,44 @@ message OperationParams { } message GetOperationRequest { - string id = 1 [(required) = true]; + string id = 1 [(required) = true]; } message GetOperationResponse { Operation operation = 1; } -message CancelOperationRequest { - string id = 1 [(required) = true]; -} - -message CancelOperationResponse { - StatusIds.StatusCode status = 1; - repeated Ydb.Issue.IssueMessage issues = 2; -} - -message ForgetOperationRequest { - string id = 1 [(required) = true]; -} - -message ForgetOperationResponse { - StatusIds.StatusCode status = 1; - repeated Ydb.Issue.IssueMessage issues = 2; -} - -message ListOperationsRequest { - string kind = 1 [(required) = true]; - uint64 page_size = 2; - string page_token = 3; -} - -message ListOperationsResponse { - StatusIds.StatusCode status = 1; - repeated Ydb.Issue.IssueMessage issues = 2; - repeated Operation operations = 3; - string next_page_token = 4; -} - +message CancelOperationRequest { + string id = 1 [(required) = true]; +} + +message CancelOperationResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; +} + +message ForgetOperationRequest { + string id = 1 [(required) = true]; +} + +message ForgetOperationResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; +} + +message ListOperationsRequest { + string kind = 1 [(required) = true]; + uint64 page_size = 2; + string page_token = 3; +} + +message ListOperationsResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; + repeated Operation operations = 3; + string next_page_token = 4; +} + message Operation { // Identifier of the operation, empty value means no active operation object is present (it was forgotten or // not created in the first place, as in SYNC operation mode). @@ -104,8 +104,8 @@ message Operation { repeated Ydb.Issue.IssueMessage issues = 4; // Result data google.protobuf.Any result = 5; - - google.protobuf.Any metadata = 6; + + google.protobuf.Any metadata = 6; // Contains information about the cost of the operation. // For completed operations, it shows the final cost of the operation. diff --git a/ydb/public/api/protos/ydb_scheme.proto b/ydb/public/api/protos/ydb_scheme.proto index a27f1ae9b37..d66b765725c 100644 --- a/ydb/public/api/protos/ydb_scheme.proto +++ b/ydb/public/api/protos/ydb_scheme.proto @@ -56,7 +56,7 @@ message Entry { BLOCK_STORE_VOLUME = 6; COORDINATION_NODE = 7; SEQUENCE = 15; - REPLICATION = 16; + REPLICATION = 16; } // Name of scheme entry (dir2 of /dir1/dir2) @@ -65,13 +65,13 @@ message Entry { string owner = 2; Type type = 5; repeated Permissions effective_permissions = 6; - repeated Permissions permissions = 7; - - // Size of entry in bytes. Currently filled for: - // - TABLE; - // - DATABASE. - // Empty (zero) in other cases. - uint64 size_bytes = 8; + repeated Permissions permissions = 7; + + // Size of entry in bytes. Currently filled for: + // - TABLE; + // - DATABASE. + // Empty (zero) in other cases. + uint64 size_bytes = 8; } message ListDirectoryResult { diff --git a/ydb/public/api/protos/ydb_status_codes.proto b/ydb/public/api/protos/ydb_status_codes.proto index 8cbd9e98241..4c500dc43ac 100644 --- a/ydb/public/api/protos/ydb_status_codes.proto +++ b/ydb/public/api/protos/ydb_status_codes.proto @@ -25,7 +25,7 @@ message StatusIds { SESSION_EXPIRED = 400150; CANCELLED = 400160; UNDETERMINED = 400170; - UNSUPPORTED = 400180; + UNSUPPORTED = 400180; SESSION_BUSY = 400190; } // reserved range [401000, 402999] for internal client status diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 8c38024169b..c8063d9475c 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -11,7 +11,7 @@ import "ydb/public/api/protos/ydb_scheme.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_formats.proto"; -import "google/protobuf/empty.proto"; +import "google/protobuf/empty.proto"; import "google/protobuf/timestamp.proto"; package Ydb.Table; @@ -47,9 +47,9 @@ message DeleteSessionResponse { message GlobalIndex { } -message GlobalAsyncIndex { -} - +message GlobalAsyncIndex { +} + // Represent secondary index message TableIndex { // Name of index @@ -59,7 +59,7 @@ message TableIndex { // Type of index oneof type { GlobalIndex global_index = 3; - GlobalAsyncIndex global_async_index = 4; + GlobalAsyncIndex global_async_index = 4; } // list of columns content to be copied in to index table repeated string data_columns = 5; @@ -81,13 +81,13 @@ message TableIndexDescription { // Type of index oneof type { GlobalIndex global_index = 3; - GlobalAsyncIndex global_async_index = 5; + GlobalAsyncIndex global_async_index = 5; } Status status = 4; // list of columns content to be copied in to index table repeated string data_columns = 6; - // Size of index data in bytes - uint64 size_bytes = 7; + // Size of index data in bytes + uint64 size_bytes = 7; } // State of index building operation @@ -250,65 +250,65 @@ message ColumnMeta { string family = 3; } -message DateTypeColumnModeSettings { - // The row will be considered as expired at the moment of time, when the value - // stored in <column_name> is less than or equal to the current time (in epoch - // time format), and <expire_after_seconds> has passed since that moment; - // i.e. the expiration threshold is the value of <column_name> plus <expire_after_seconds>. - - // The column type must be a date type - string column_name = 1; - - uint32 expire_after_seconds = 2; -} - -message ValueSinceUnixEpochModeSettings { - // Same as DateTypeColumnModeSettings (above), but useful when type of the - // value stored in <column_name> is not a date type. - - enum Unit { - UNIT_UNSPECIFIED = 0; - UNIT_SECONDS = 1; - UNIT_MILLISECONDS = 2; - UNIT_MICROSECONDS = 3; - UNIT_NANOSECONDS = 4; - } - - // The column type must be one of: - // - Uint32 - // - Uint64 - // - DyNumber - string column_name = 1; - - // Interpretation of the value stored in <column_name> - Unit column_unit = 2; - - // This option is always interpreted as seconds regardless of the - // <column_unit> value. - uint32 expire_after_seconds = 3; -} - -message TtlSettings { - oneof mode { - DateTypeColumnModeSettings date_type_column = 1; - ValueSinceUnixEpochModeSettings value_since_unix_epoch = 2; - } - - // There is no guarantee that expired row will be deleted immediately upon - // expiration. There may be a delay between the time a row expires and the - // time that server deletes the row from the table. - - // Ttl periodically runs background removal operations (BRO) on table's partitions. - // By default, there is: - // - no more than one BRO on the table; - // - BRO is started no more than once an hour on the same partition. - // Use options below to change that behavior. - - // How often to run BRO on the same partition. - // BRO will not be started more often, but may be started less often. - uint32 run_interval_seconds = 3; -} - +message DateTypeColumnModeSettings { + // The row will be considered as expired at the moment of time, when the value + // stored in <column_name> is less than or equal to the current time (in epoch + // time format), and <expire_after_seconds> has passed since that moment; + // i.e. the expiration threshold is the value of <column_name> plus <expire_after_seconds>. + + // The column type must be a date type + string column_name = 1; + + uint32 expire_after_seconds = 2; +} + +message ValueSinceUnixEpochModeSettings { + // Same as DateTypeColumnModeSettings (above), but useful when type of the + // value stored in <column_name> is not a date type. + + enum Unit { + UNIT_UNSPECIFIED = 0; + UNIT_SECONDS = 1; + UNIT_MILLISECONDS = 2; + UNIT_MICROSECONDS = 3; + UNIT_NANOSECONDS = 4; + } + + // The column type must be one of: + // - Uint32 + // - Uint64 + // - DyNumber + string column_name = 1; + + // Interpretation of the value stored in <column_name> + Unit column_unit = 2; + + // This option is always interpreted as seconds regardless of the + // <column_unit> value. + uint32 expire_after_seconds = 3; +} + +message TtlSettings { + oneof mode { + DateTypeColumnModeSettings date_type_column = 1; + ValueSinceUnixEpochModeSettings value_since_unix_epoch = 2; + } + + // There is no guarantee that expired row will be deleted immediately upon + // expiration. There may be a delay between the time a row expires and the + // time that server deletes the row from the table. + + // Ttl periodically runs background removal operations (BRO) on table's partitions. + // By default, there is: + // - no more than one BRO on the table; + // - BRO is started no more than once an hour on the same partition. + // Use options below to change that behavior. + + // How often to run BRO on the same partition. + // BRO will not be started more often, but may be started less often. + uint32 run_interval_seconds = 3; +} + message StorageSettings { // This specifies internal channel 0 commit log storage pool // Fastest available storage recommended, negligible amounts of short-lived data @@ -408,13 +408,13 @@ message CreateTableRequest { // List of secondary indexes repeated TableIndex indexes = 7; // Table rows time to live settings - TtlSettings ttl_settings = 8; + TtlSettings ttl_settings = 8; // Storage settings for table StorageSettings storage_settings = 9; // Column families repeated ColumnFamily column_families = 10; - // Attributes. Total size is limited to 10 KB. - map<string, string> attributes = 11 [(map_key).length.range = {min: 1, max: 100}, (length).range = {min: 1, max: 4096}]; + // Attributes. Total size is limited to 10 KB. + map<string, string> attributes = 11 [(map_key).length.range = {min: 1, max: 100}, (length).range = {min: 1, max: 4096}]; // Predefined named set of settings for table compaction ["default", "small_table", "log_table"]. string compaction_policy = 12; // Either one of the following partitions options can be specified @@ -465,10 +465,10 @@ message AlterTableRequest { // Columns to alter repeated ColumnMeta alter_columns = 6; // Setup or remove time to live settings - oneof ttl_action { - TtlSettings set_ttl_settings = 7; - google.protobuf.Empty drop_ttl_settings = 8; - } + oneof ttl_action { + TtlSettings set_ttl_settings = 7; + google.protobuf.Empty drop_ttl_settings = 8; + } // Add secondary indexes repeated TableIndex add_indexes = 9; // Remove secondary indexes @@ -478,9 +478,9 @@ message AlterTableRequest { // Add/alter column families repeated ColumnFamily add_column_families = 12; repeated ColumnFamily alter_column_families = 13; - // Alter attributes. Leave the value blank to drop an attribute. - // Cannot be used in combination with other fields (except session_id and path) at the moment. - map<string, string> alter_attributes = 14 [(map_key).length.range = {min: 1, max: 100}, (length).le = 4096]; + // Alter attributes. Leave the value blank to drop an attribute. + // Cannot be used in combination with other fields (except session_id and path) at the moment. + map<string, string> alter_attributes = 14 [(map_key).length.range = {min: 1, max: 100}, (length).le = 4096]; // Set predefined named set of settings for table compaction ["default", "small_table", "log_table"]. // Set "default" to use default preset. string set_compaction_policy = 15; @@ -588,14 +588,14 @@ message DescribeTableResult { repeated TableIndexDescription indexes = 5; // Statistics of table TableStats table_stats = 6; - // TTL params - TtlSettings ttl_settings = 7; + // TTL params + TtlSettings ttl_settings = 7; // Storage settings for table StorageSettings storage_settings = 8; // Column families repeated ColumnFamily column_families = 9; - // Attributes - map<string, string> attributes = 10; + // Attributes + map<string, string> attributes = 10; // Predefined named set of settings for table compaction reserved 11; // compaction_policy // Partitioning settings for table diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp index 485747c92e1..2b3c28cf3fc 100644 --- a/ydb/public/lib/deprecated/kicli/configurator.cpp +++ b/ydb/public/lib/deprecated/kicli/configurator.cpp @@ -42,10 +42,10 @@ TConfigurationResult TNodeConfigurator::SyncGetNodeConfig(ui32 nodeId, const TString &host, const TString &tenant, const TString &nodeType, - const TString& domain, - const TString& token) const + const TString& domain, + const TString& token) const { - auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token); + auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token); auto result = future.GetValue(TDuration::Max()); return TConfigurationResult(result); } diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index bb9871c8644..9dc472d2314 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -578,7 +578,7 @@ public: OlapStore, OlapTable, Sequence, - Replication, + Replication, }; TSchemaObject(TSchemaObject&&) = default; @@ -693,8 +693,8 @@ public: const TString &host, const TString &tenant, const TString &nodeType, - const TString& domain = "", - const TString& token = "") const; + const TString& domain = "", + const TString& token = "") const; private: TNodeConfigurator(TKikimr& kikimr); @@ -798,8 +798,8 @@ protected: const TString &host, const TString &tenant, const TString &nodeType, - const TString& domain, - const TString& token = TString()); + const TString& domain, + const TString& token = TString()); template <typename T> static void DumpRequest(const T& pb) { if (DUMP_REQUESTS) { diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp index 3abf847868f..4156d72d4b3 100644 --- a/ydb/public/lib/deprecated/kicli/kikimr.cpp +++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp @@ -598,8 +598,8 @@ NThreading::TFuture<TResult> TKikimr::GetNodeConfig(ui32 nodeId, const TString &host, const TString &tenant, const TString &nodeType, - const TString& domain, - const TString& token) + const TString& domain, + const TString& token) { TAutoPtr<NMsgBusProxy::TBusConsoleRequest> request = new NMsgBusProxy::TBusConsoleRequest; auto &node = *request->Record.MutableGetNodeConfigRequest()->MutableNode(); @@ -609,8 +609,8 @@ NThreading::TFuture<TResult> TKikimr::GetNodeConfig(ui32 nodeId, node.SetNodeType(nodeType); if (domain) request->Record.SetDomainName(domain); - if (token) - request->Record.SetSecurityToken(token); + if (token) + request->Record.SetSecurityToken(token); return ExecuteRequest(request.Release()); } diff --git a/ydb/public/lib/deprecated/kicli/schema.cpp b/ydb/public/lib/deprecated/kicli/schema.cpp index c4167dba3ba..d1ee0384f8b 100644 --- a/ydb/public/lib/deprecated/kicli/schema.cpp +++ b/ydb/public/lib/deprecated/kicli/schema.cpp @@ -116,9 +116,9 @@ void TSchemaObject::Drop() { case EPathType::Sequence: drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropSequence); break; - case EPathType::Replication: - drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication); - break; + case EPathType::Replication: + drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropReplication); + break; case EPathType::Unknown: case EPathType::SubDomain: case EPathType::RtmrVolume: @@ -200,8 +200,8 @@ static TSchemaObject::EPathType GetType(const NKikimrSchemeOp::TDirEntry& entry) return TSchemaObject::EPathType::OlapTable; case NKikimrSchemeOp::EPathTypeSequence: return TSchemaObject::EPathType::Sequence; - case NKikimrSchemeOp::EPathTypeReplication: - return TSchemaObject::EPathType::Replication; + case NKikimrSchemeOp::EPathTypeReplication: + return TSchemaObject::EPathType::Replication; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeExtSubDomain: case NKikimrSchemeOp::EPathTypeCdcStream: diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp index 3e0393ae9e4..9bd2da5107a 100644 --- a/ydb/public/lib/experimental/ydb_logstore.cpp +++ b/ydb/public/lib/experimental/ydb_logstore.cpp @@ -17,17 +17,17 @@ namespace NLogStore { TMaybe<TTtlSettings> TtlSettingsFromProto(const Ydb::Table::TtlSettings& proto) { switch (proto.mode_case()) { - case Ydb::Table::TtlSettings::kDateTypeColumn: - return TTtlSettings( - proto.date_type_column(), - proto.run_interval_seconds() - ); - - case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: - return TTtlSettings( - proto.value_since_unix_epoch(), - proto.run_interval_seconds() - ); + case Ydb::Table::TtlSettings::kDateTypeColumn: + return TTtlSettings( + proto.date_type_column(), + proto.run_interval_seconds() + ); + + case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: + return TTtlSettings( + proto.value_since_unix_epoch(), + proto.run_interval_seconds() + ); default: break; diff --git a/ydb/public/lib/operation_id/operation_id.cpp b/ydb/public/lib/operation_id/operation_id.cpp index ecdae5f926d..725da690f41 100644 --- a/ydb/public/lib/operation_id/operation_id.cpp +++ b/ydb/public/lib/operation_id/operation_id.cpp @@ -1,15 +1,15 @@ #include "operation_id.h" #include <google/protobuf/message.h> - + #include <library/cpp/cgiparam/cgiparam.h> #include <library/cpp/uri/uri.h> - + #include <util/generic/yexception.h> #include <util/string/cast.h> namespace NKikimr { -namespace NOperationId { +namespace NOperationId { using namespace NUri; @@ -33,12 +33,12 @@ TString ProtoToString(const Ydb::TOperationId& proto) { case Ydb::TOperationId::CMS_REQUEST: res << "ydb://cmsrequest"; break; - case Ydb::TOperationId::EXPORT: - res << "ydb://export"; - break; - case Ydb::TOperationId::IMPORT: - res << "ydb://import"; - break; + case Ydb::TOperationId::EXPORT: + res << "ydb://export"; + break; + case Ydb::TOperationId::IMPORT: + res << "ydb://import"; + break; case Ydb::TOperationId::BUILD_INDEX: res << "ydb://buildindex"; break; @@ -83,16 +83,16 @@ TString ProtoToString(const Ydb::TOperationId& proto) { return res.Str(); } -TOperationId::TOperationId() { - SetKind(Ydb::TOperationId::UNUSED); -} - -TOperationId::TOperationId(const TString &string, bool allowEmpty) { - if (allowEmpty && string.empty()) { - SetKind(Ydb::TOperationId::UNUSED); - return; - } - +TOperationId::TOperationId() { + SetKind(Ydb::TOperationId::UNUSED); +} + +TOperationId::TOperationId(const TString &string, bool allowEmpty) { + if (allowEmpty && string.empty()) { + SetKind(Ydb::TOperationId::UNUSED); + return; + } + TUri uri; TState::EParsed er = uri.Parse(string, TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible); if (er != TState::ParsedOK) { @@ -104,15 +104,15 @@ TOperationId::TOperationId(const TString &string, bool allowEmpty) { ythrow yexception() << "Invalid path length"; } - int kind; + int kind; if (!TryFromString(path, kind)) { ythrow yexception() << "Unable to cast \"kind\" field: " << path; } - if (!EKind_IsValid(kind)) { - ythrow yexception() << "Invalid operation kind: " << kind; - } - + if (!EKind_IsValid(kind)) { + ythrow yexception() << "Invalid operation kind: " << kind; + } + SetKind(static_cast<Ydb::TOperationId::EKind>(kind)); const TString& query = uri.PrintS(TField::FlagQuery); @@ -136,40 +136,40 @@ const TVector<const TString*>& TOperationId::GetValue(const TString &key) const ythrow yexception() << "Unable to find key: " << key; } -TString TOperationId::GetSubKind() const { - auto it = Index_.find("kind"); - if (it == Index_.end()) { - return TString(); - } - - if (it->second.size() != 1) { - ythrow yexception() << "Unable to retreive sub-kind"; - } - - return *it->second.at(0); -} - +TString TOperationId::GetSubKind() const { + auto it = Index_.find("kind"); + if (it == Index_.end()) { + return TString(); + } + + if (it->second.size() != 1) { + ythrow yexception() << "Unable to retreive sub-kind"; + } + + return *it->second.at(0); +} + void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value) { auto data = proto.AddData(); data->SetKey(key); data->SetValue(value); } -Ydb::TOperationId::EKind ParseKind(const TStringBuf value) { - if (value.StartsWith("export")) { - return Ydb::TOperationId::EXPORT; - } - - if (value.StartsWith("import")) { - return Ydb::TOperationId::IMPORT; - } - +Ydb::TOperationId::EKind ParseKind(const TStringBuf value) { + if (value.StartsWith("export")) { + return Ydb::TOperationId::EXPORT; + } + + if (value.StartsWith("import")) { + return Ydb::TOperationId::IMPORT; + } + if (value.StartsWith("buildindex")) { return Ydb::TOperationId::BUILD_INDEX; } - return Ydb::TOperationId::UNUSED; -} - -} // namespace NOperationId + return Ydb::TOperationId::UNUSED; +} + +} // namespace NOperationId } // namespace NKikimr diff --git a/ydb/public/lib/operation_id/operation_id.h b/ydb/public/lib/operation_id/operation_id.h index 8fc8701f4b9..23f91140d16 100644 --- a/ydb/public/lib/operation_id/operation_id.h +++ b/ydb/public/lib/operation_id/operation_id.h @@ -7,15 +7,15 @@ #include <util/generic/vector.h> namespace NKikimr { -namespace NOperationId { +namespace NOperationId { class TOperationId : public Ydb::TOperationId { public: - TOperationId(); - explicit TOperationId(const TString& string, bool allowEmpty = false); + TOperationId(); + explicit TOperationId(const TString& string, bool allowEmpty = false); const TVector<const TString*>& GetValue(const TString& key) const; - TString GetSubKind() const; - + TString GetSubKind() const; + private: THashMap<TString, TVector<const TString*>> Index_; }; @@ -23,7 +23,7 @@ private: TString ProtoToString(const Ydb::TOperationId& proto); void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const TString& value); void AddOptionalValue(Ydb::TOperationId& proto, const TString& key, const char* value, size_t size); -Ydb::TOperationId::EKind ParseKind(const TStringBuf value); +Ydb::TOperationId::EKind ParseKind(const TStringBuf value); -} // namespace NOperationId +} // namespace NOperationId } // namespace NKikimr diff --git a/ydb/public/lib/operation_id/operation_id_ut.cpp b/ydb/public/lib/operation_id/operation_id_ut.cpp index 52e39a8307a..64fbedce4bf 100644 --- a/ydb/public/lib/operation_id/operation_id_ut.cpp +++ b/ydb/public/lib/operation_id/operation_id_ut.cpp @@ -4,7 +4,7 @@ #include <library/cpp/testing/unittest/registar.h> namespace NKikimr { -namespace NOperationId { +namespace NOperationId { Y_UNIT_TEST_SUITE(OperationIdTest) { Y_UNIT_TEST(ConvertKindOnly) { @@ -48,5 +48,5 @@ Y_UNIT_TEST_SUITE(OperationIdTest) { } } -} // namespace NOperationId +} // namespace NOperationId } // namespace NKikimr diff --git a/ydb/public/lib/operation_id/protos/operation_id.proto b/ydb/public/lib/operation_id/protos/operation_id.proto index c5bf39652f5..b6534e794ba 100644 --- a/ydb/public/lib/operation_id/protos/operation_id.proto +++ b/ydb/public/lib/operation_id/protos/operation_id.proto @@ -10,16 +10,16 @@ message TOperationId { SESSION_YQL = 3; PREPARED_QUERY_ID = 4; CMS_REQUEST = 5; - EXPORT = 6; + EXPORT = 6; BUILD_INDEX = 7; - IMPORT = 8; + IMPORT = 8; } - + message TData { string Key = 1; string Value = 2; } - + EKind Kind = 1; repeated TData Data = 3; } diff --git a/ydb/public/lib/operation_id/protos/ya.make b/ydb/public/lib/operation_id/protos/ya.make index 6e4afbb9551..e1b15b77c11 100644 --- a/ydb/public/lib/operation_id/protos/ya.make +++ b/ydb/public/lib/operation_id/protos/ya.make @@ -1,15 +1,15 @@ -PROTO_LIBRARY() - +PROTO_LIBRARY() + OWNER(g:kikimr) - -IF (OS_WINDOWS) - NO_OPTIMIZE_PY_PROTOS() -ENDIF() - -SRCS( - operation_id.proto -) - + +IF (OS_WINDOWS) + NO_OPTIMIZE_PY_PROTOS() +ENDIF() + +SRCS( + operation_id.proto +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/ydb/public/lib/operation_id/ut/ya.make b/ydb/public/lib/operation_id/ut/ya.make index 75c72cd45ef..bff26b3e09d 100644 --- a/ydb/public/lib/operation_id/ut/ya.make +++ b/ydb/public/lib/operation_id/ut/ya.make @@ -1,22 +1,22 @@ UNITTEST_FOR(ydb/public/lib/operation_id) - -OWNER( - dcherednik - g:kikimr -) - -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - SIZE(MEDIUM) -ENDIF() - -SRCS( - operation_id_ut.cpp -) - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +SRCS( + operation_id_ut.cpp +) + +PEERDIR( library/cpp/testing/unittest -) - -END() +) + +END() diff --git a/ydb/public/lib/operation_id/ya.make b/ydb/public/lib/operation_id/ya.make index 74e3d3cbff1..d8a249f59a4 100644 --- a/ydb/public/lib/operation_id/ya.make +++ b/ydb/public/lib/operation_id/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - operation_id.cpp -) - -PEERDIR( - contrib/libs/protobuf +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + operation_id.cpp +) + +PEERDIR( + contrib/libs/protobuf library/cpp/cgiparam library/cpp/uri ydb/public/lib/operation_id/protos -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/lib/ya.make b/ydb/public/lib/ya.make index 73ddcf46dd1..a4f529b44bc 100644 --- a/ydb/public/lib/ya.make +++ b/ydb/public/lib/ya.make @@ -7,7 +7,7 @@ RECURSE( idx_test json_value jwt - operation_id + operation_id scheme_types value ydb_cli diff --git a/ydb/public/lib/ydb_cli/commands/ydb_common.h b/ydb/public/lib/ydb_cli/commands/ydb_common.h index d4e793e177e..7561c6476e9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_common.h @@ -37,11 +37,11 @@ inline void ThrowOnError(NYdb::TStatus status) { } } -inline void ThrowOnError(const NYdb::TOperation& operation) { +inline void ThrowOnError(const NYdb::TOperation& operation) { if (!operation.Ready()) return; - ThrowOnError(operation.Status()); + ThrowOnError(operation.Status()); +} + } - } -} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp index 04ce2be0d3e..f997343dc85 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp @@ -1,324 +1,324 @@ -#include "ydb_service_export.h" - +#include "ydb_service_export.h" + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/lib/ydb_cli/common/normalize_path.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> #include <ydb/public/lib/ydb_cli/common/recursive_list.h> - -#include <util/generic/serialized_enum.h> -#include <util/string/builder.h> - -namespace NYdb { -namespace NConsoleClient { - -static const TString AppendPrefix = "<append=true>"; - -namespace { - - const char slashC = '/'; - const TStringBuf slash(&slashC, 1); - - bool FilterTables(const NScheme::TSchemeEntry& entry) { - return entry.Type == NScheme::ESchemeEntryType::Table; - } - - TVector<std::pair<TString, TString>> ExpandItem(NScheme::TSchemeClient& client, TStringBuf srcPath, TStringBuf dstPath) { - // cut trailing slash - srcPath.ChopSuffix(slash); - dstPath.ChopSuffix(slash); - - const auto ret = RecursiveList(client, TString{srcPath}, TRecursiveListSettings().Filter(&FilterTables)); - ThrowOnError(ret.Status); - - if (ret.Entries.size() == 1 && srcPath == ret.Entries[0].Name) { + +#include <util/generic/serialized_enum.h> +#include <util/string/builder.h> + +namespace NYdb { +namespace NConsoleClient { + +static const TString AppendPrefix = "<append=true>"; + +namespace { + + const char slashC = '/'; + const TStringBuf slash(&slashC, 1); + + bool FilterTables(const NScheme::TSchemeEntry& entry) { + return entry.Type == NScheme::ESchemeEntryType::Table; + } + + TVector<std::pair<TString, TString>> ExpandItem(NScheme::TSchemeClient& client, TStringBuf srcPath, TStringBuf dstPath) { + // cut trailing slash + srcPath.ChopSuffix(slash); + dstPath.ChopSuffix(slash); + + const auto ret = RecursiveList(client, TString{srcPath}, TRecursiveListSettings().Filter(&FilterTables)); + ThrowOnError(ret.Status); + + if (ret.Entries.size() == 1 && srcPath == ret.Entries[0].Name) { return {{TString{srcPath}, TString{dstPath}}}; - } - - TVector<std::pair<TString, TString>> result; - for (const auto& table : ret.Entries) { - result.emplace_back(table.Name, TStringBuilder() << dstPath << TStringBuf(table.Name).RNextTok(srcPath)); - } - - return result; - } - - template <typename TSettings> - void ExpandItems(NScheme::TSchemeClient& client, TSettings& settings, const TVector<TRegExMatch>& exclusions) { - auto isExclusion = [&exclusions](const char* str) -> bool { - for (const auto& pattern : exclusions) { - if (pattern.Match(str)) { - return true; - } - } - - return false; - }; - - auto items(std::move(settings.Item_)); - for (const auto& item : items) { - for (const auto& [src, dst] : ExpandItem(client, item.Src, item.Dst)) { - if (isExclusion(src.c_str())) { - continue; - } - - settings.AppendItem({src, dst}); - } - } - } - -} // anonymous namespace - + } + + TVector<std::pair<TString, TString>> result; + for (const auto& table : ret.Entries) { + result.emplace_back(table.Name, TStringBuilder() << dstPath << TStringBuf(table.Name).RNextTok(srcPath)); + } + + return result; + } + + template <typename TSettings> + void ExpandItems(NScheme::TSchemeClient& client, TSettings& settings, const TVector<TRegExMatch>& exclusions) { + auto isExclusion = [&exclusions](const char* str) -> bool { + for (const auto& pattern : exclusions) { + if (pattern.Match(str)) { + return true; + } + } + + return false; + }; + + auto items(std::move(settings.Item_)); + for (const auto& item : items) { + for (const auto& [src, dst] : ExpandItem(client, item.Src, item.Dst)) { + if (isExclusion(src.c_str())) { + continue; + } + + settings.AppendItem({src, dst}); + } + } + } + +} // anonymous namespace + TCommandExport::TCommandExport(bool useExportToYt) - : TClientCommandTree("export", {}, "Export service operations") -{ + : TClientCommandTree("export", {}, "Export service operations") +{ if (useExportToYt) { AddCommand(std::make_unique<TCommandExportToYt>()); } AddCommand(std::make_unique<TCommandExportToS3>()); -} - -/// YT -TCommandExportToYt::TCommandExportToYt() - : TYdbOperationCommand("yt", {}, "Create export to YT") -{ - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - TItem::DefineFields({ - {"Source", {{"source", "src", "s"}, "Database path to a directory or a table to be exported", true}}, - {"Destination", {{"destination", "dst", "d"}, "Path to a table or a directory in YT", true}}, - {"Append", {{"append", "a"}, TStringBuilder() << "Append rows to existent YT table (default: " - << colors.CyanColor() << "true" << colors.OldColor() << ")", false}} - }); -} - -void TCommandExportToYt::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - TStringBuilder proxyHelp; - proxyHelp << "YT proxy to connect to" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"YT_PROXY\" environment variable"; - config.Opts->AddLongOption("proxy", proxyHelp) - .RequiredArgument("PROXY"); - - TStringBuilder tokenHelp; - tokenHelp << "OAuth token" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"YT_TOKEN\" environment variable" << Endl - << " 3. \"" << YtTokenFile << "\" file"; - config.Opts->AddLongOption("token", tokenHelp) - .RequiredArgument("TOKEN"); - - TStringBuilder itemHelp; - itemHelp << "[At least one] Item specification" << Endl - << " Possible property names:" << Endl - << TItem::FormatHelp(2); - config.Opts->AddLongOption("item", itemHelp) - .RequiredArgument("PROPERTY=VALUE,..."); - - config.Opts->AddLongOption("exclude", "Pattern (PCRE) for paths excluded from export operation") - .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { - ExclusionPatterns.emplace_back(TRegExMatch(arg)); - }); - - config.Opts->AddLongOption("description", "Textual description of export operation") - .RequiredArgument("STRING").StoreResult(&Description); - - config.Opts->AddLongOption("retries", "Number of retries") - .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); - - config.Opts->AddLongOption("use-type-v3", "Use YT's type_v3") - .NoArgument().StoreTrue(&UseTypeV3); - - AddJsonOption(config); +} + +/// YT +TCommandExportToYt::TCommandExportToYt() + : TYdbOperationCommand("yt", {}, "Create export to YT") +{ + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + TItem::DefineFields({ + {"Source", {{"source", "src", "s"}, "Database path to a directory or a table to be exported", true}}, + {"Destination", {{"destination", "dst", "d"}, "Path to a table or a directory in YT", true}}, + {"Append", {{"append", "a"}, TStringBuilder() << "Append rows to existent YT table (default: " + << colors.CyanColor() << "true" << colors.OldColor() << ")", false}} + }); +} + +void TCommandExportToYt::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + TStringBuilder proxyHelp; + proxyHelp << "YT proxy to connect to" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"YT_PROXY\" environment variable"; + config.Opts->AddLongOption("proxy", proxyHelp) + .RequiredArgument("PROXY"); + + TStringBuilder tokenHelp; + tokenHelp << "OAuth token" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"YT_TOKEN\" environment variable" << Endl + << " 3. \"" << YtTokenFile << "\" file"; + config.Opts->AddLongOption("token", tokenHelp) + .RequiredArgument("TOKEN"); + + TStringBuilder itemHelp; + itemHelp << "[At least one] Item specification" << Endl + << " Possible property names:" << Endl + << TItem::FormatHelp(2); + config.Opts->AddLongOption("item", itemHelp) + .RequiredArgument("PROPERTY=VALUE,..."); + + config.Opts->AddLongOption("exclude", "Pattern (PCRE) for paths excluded from export operation") + .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { + ExclusionPatterns.emplace_back(TRegExMatch(arg)); + }); + + config.Opts->AddLongOption("description", "Textual description of export operation") + .RequiredArgument("STRING").StoreResult(&Description); + + config.Opts->AddLongOption("retries", "Number of retries") + .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); + + config.Opts->AddLongOption("use-type-v3", "Use YT's type_v3") + .NoArgument().StoreTrue(&UseTypeV3); + + AddJsonOption(config); AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); config.Opts->MutuallyExclusive("json", "format"); -} - -void TCommandExportToYt::Parse(TConfig& config) { - TClientCommand::Parse(config); +} + +void TCommandExportToYt::Parse(TConfig& config) { + TClientCommand::Parse(config); ParseFormats(); - - ParseYtProxy(config, "proxy"); - ParseYtToken(config, "token"); - - Items = TItem::Parse(config, "item"); - if (Items.empty()) { - throw TMissUseException() << "At least one item should be provided"; - } - - for (auto& item : Items) { - NConsoleClient::AdjustPath(item.Source, config); - - const bool hasAppendPrefix = item.Destination.StartsWith(AppendPrefix); - if (item.Append && !hasAppendPrefix) { - item.Destination.prepend(AppendPrefix); - } else if (!item.Append && hasAppendPrefix) { - Cerr << "warning: 'Append' option is false, but path has " - << "'" << AppendPrefix << "' prefix: " << item.Destination << Endl; - } - } -} - -int TCommandExportToYt::Run(TConfig& config) { - using namespace NExport; - using namespace NScheme; - - TExportToYtSettings settings = FillSettings(TExportToYtSettings()); - - settings.Host(YtHost); - settings.Port(YtPort); - settings.Token(YtToken); - - for (const auto& item : Items) { - settings.AppendItem({item.Source, item.Destination}); - } - - if (Description) { - settings.Description(Description); - } - - settings.NumberOfRetries(NumberOfRetries); - settings.UseTypeV3(UseTypeV3); - - const TDriver driver = CreateDriver(config); - - TSchemeClient schemeClient(driver); - ExpandItems(schemeClient, settings, ExclusionPatterns); - - TExportClient client(driver); - TExportToYtResponse response = client.ExportToYt(std::move(settings)).GetValueSync(); - ThrowOnError(response); - PrintOperation(response, OutputFormat); - - return EXIT_SUCCESS; -} - -/// S3 -TCommandExportToS3::TCommandExportToS3() - : TYdbOperationCommand("s3", {}, "Create export to S3") -{ - TItem::DefineFields({ - {"Source", {{"source", "src", "s"}, "Database path to a directory or a table to be exported", true}}, - {"Destination", {{"destination", "dst", "d"}, "S3 object key prefix", true}}, - }); -} - -void TCommandExportToS3::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.Opts->AddLongOption("s3-endpoint", "S3 endpoint to connect to") - .Required().RequiredArgument("ENDPOINT").StoreResult(&AwsEndpoint); - - config.Opts->AddLongOption("scheme", "S3 endpoint scheme") - .RequiredArgument("SCHEME").StoreResult(&AwsScheme).DefaultValue(AwsScheme); - - TStringBuilder storageClassHelp; - storageClassHelp << "S3 storage class" << Endl - << " Available options:" << Endl; - for (auto value : GetEnumAllValues<EStorageClass>()) { - storageClassHelp << " - " << value << Endl; - } - config.Opts->AddLongOption("storage-class", storageClassHelp) - .RequiredArgument("STORAGE_CLASS").StoreResult(&AwsStorageClass).DefaultValue(AwsStorageClass); - - config.Opts->AddLongOption("bucket", "S3 bucket") - .Required().RequiredArgument("BUCKET").StoreResult(&AwsBucket); - - TStringBuilder accessKeyHelp; - accessKeyHelp << "Access key id" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"AWS_ACCESS_KEY_ID\" environment variable" << Endl - << " 3. \"aws_access_key_id\" key in \"" << AwsCredentialsFile << "\" file"; - config.Opts->AddLongOption("access-key", accessKeyHelp) - .RequiredArgument("STRING"); - - TStringBuilder secretKeyHelp; - secretKeyHelp << "Secret key" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"AWS_SECRET_ACCESS_KEY\" environment variable" << Endl - << " 3. \"aws_secret_access_key\" key in \"" << AwsCredentialsFile << "\" file"; - config.Opts->AddLongOption("secret-key", secretKeyHelp) - .RequiredArgument("STRING"); - - TStringBuilder itemHelp; - itemHelp << "[At least one] Item specification" << Endl - << " Possible property names:" << Endl - << TItem::FormatHelp(2); - config.Opts->AddLongOption("item", itemHelp) - .RequiredArgument("PROPERTY=VALUE,..."); - - config.Opts->AddLongOption("exclude", "Pattern (PCRE) for paths excluded from export operation") - .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { - ExclusionPatterns.emplace_back(TRegExMatch(arg)); - }); - - config.Opts->AddLongOption("description", "Textual description of export operation") - .RequiredArgument("STRING").StoreResult(&Description); - - config.Opts->AddLongOption("retries", "Number of retries") - .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); - - AddJsonOption(config); + + ParseYtProxy(config, "proxy"); + ParseYtToken(config, "token"); + + Items = TItem::Parse(config, "item"); + if (Items.empty()) { + throw TMissUseException() << "At least one item should be provided"; + } + + for (auto& item : Items) { + NConsoleClient::AdjustPath(item.Source, config); + + const bool hasAppendPrefix = item.Destination.StartsWith(AppendPrefix); + if (item.Append && !hasAppendPrefix) { + item.Destination.prepend(AppendPrefix); + } else if (!item.Append && hasAppendPrefix) { + Cerr << "warning: 'Append' option is false, but path has " + << "'" << AppendPrefix << "' prefix: " << item.Destination << Endl; + } + } +} + +int TCommandExportToYt::Run(TConfig& config) { + using namespace NExport; + using namespace NScheme; + + TExportToYtSettings settings = FillSettings(TExportToYtSettings()); + + settings.Host(YtHost); + settings.Port(YtPort); + settings.Token(YtToken); + + for (const auto& item : Items) { + settings.AppendItem({item.Source, item.Destination}); + } + + if (Description) { + settings.Description(Description); + } + + settings.NumberOfRetries(NumberOfRetries); + settings.UseTypeV3(UseTypeV3); + + const TDriver driver = CreateDriver(config); + + TSchemeClient schemeClient(driver); + ExpandItems(schemeClient, settings, ExclusionPatterns); + + TExportClient client(driver); + TExportToYtResponse response = client.ExportToYt(std::move(settings)).GetValueSync(); + ThrowOnError(response); + PrintOperation(response, OutputFormat); + + return EXIT_SUCCESS; +} + +/// S3 +TCommandExportToS3::TCommandExportToS3() + : TYdbOperationCommand("s3", {}, "Create export to S3") +{ + TItem::DefineFields({ + {"Source", {{"source", "src", "s"}, "Database path to a directory or a table to be exported", true}}, + {"Destination", {{"destination", "dst", "d"}, "S3 object key prefix", true}}, + }); +} + +void TCommandExportToS3::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.Opts->AddLongOption("s3-endpoint", "S3 endpoint to connect to") + .Required().RequiredArgument("ENDPOINT").StoreResult(&AwsEndpoint); + + config.Opts->AddLongOption("scheme", "S3 endpoint scheme") + .RequiredArgument("SCHEME").StoreResult(&AwsScheme).DefaultValue(AwsScheme); + + TStringBuilder storageClassHelp; + storageClassHelp << "S3 storage class" << Endl + << " Available options:" << Endl; + for (auto value : GetEnumAllValues<EStorageClass>()) { + storageClassHelp << " - " << value << Endl; + } + config.Opts->AddLongOption("storage-class", storageClassHelp) + .RequiredArgument("STORAGE_CLASS").StoreResult(&AwsStorageClass).DefaultValue(AwsStorageClass); + + config.Opts->AddLongOption("bucket", "S3 bucket") + .Required().RequiredArgument("BUCKET").StoreResult(&AwsBucket); + + TStringBuilder accessKeyHelp; + accessKeyHelp << "Access key id" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"AWS_ACCESS_KEY_ID\" environment variable" << Endl + << " 3. \"aws_access_key_id\" key in \"" << AwsCredentialsFile << "\" file"; + config.Opts->AddLongOption("access-key", accessKeyHelp) + .RequiredArgument("STRING"); + + TStringBuilder secretKeyHelp; + secretKeyHelp << "Secret key" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"AWS_SECRET_ACCESS_KEY\" environment variable" << Endl + << " 3. \"aws_secret_access_key\" key in \"" << AwsCredentialsFile << "\" file"; + config.Opts->AddLongOption("secret-key", secretKeyHelp) + .RequiredArgument("STRING"); + + TStringBuilder itemHelp; + itemHelp << "[At least one] Item specification" << Endl + << " Possible property names:" << Endl + << TItem::FormatHelp(2); + config.Opts->AddLongOption("item", itemHelp) + .RequiredArgument("PROPERTY=VALUE,..."); + + config.Opts->AddLongOption("exclude", "Pattern (PCRE) for paths excluded from export operation") + .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { + ExclusionPatterns.emplace_back(TRegExMatch(arg)); + }); + + config.Opts->AddLongOption("description", "Textual description of export operation") + .RequiredArgument("STRING").StoreResult(&Description); + + config.Opts->AddLongOption("retries", "Number of retries") + .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); + + AddJsonOption(config); AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); config.Opts->MutuallyExclusive("json", "format"); -} - -void TCommandExportToS3::Parse(TConfig& config) { - TClientCommand::Parse(config); +} + +void TCommandExportToS3::Parse(TConfig& config) { + TClientCommand::Parse(config); ParseFormats(); - - ParseAwsAccessKey(config, "access-key"); - ParseAwsSecretKey(config, "secret-key"); - - Items = TItem::Parse(config, "item"); - if (Items.empty()) { - throw TMissUseException() << "At least one item should be provided"; - } - - for (auto& item : Items) { - NConsoleClient::AdjustPath(item.Source, config); - } -} - -int TCommandExportToS3::Run(TConfig& config) { - using namespace NExport; - using namespace NScheme; - - TExportToS3Settings settings = FillSettings(TExportToS3Settings()); - - settings.Endpoint(AwsEndpoint); - settings.Scheme(AwsScheme); - settings.StorageClass(AwsStorageClass); - settings.Bucket(AwsBucket); - settings.AccessKey(AwsAccessKey); - settings.SecretKey(AwsSecretKey); - - for (const auto& item : Items) { - settings.AppendItem({item.Source, item.Destination}); - } - - if (Description) { - settings.Description(Description); - } - - settings.NumberOfRetries(NumberOfRetries); - - const TDriver driver = CreateDriver(config); - - TSchemeClient schemeClient(driver); - ExpandItems(schemeClient, settings, ExclusionPatterns); - - TExportClient client(driver); - TExportToS3Response response = client.ExportToS3(std::move(settings)).GetValueSync(); - ThrowOnError(response); - PrintOperation(response, OutputFormat); - - return EXIT_SUCCESS; -} - -} -} + + ParseAwsAccessKey(config, "access-key"); + ParseAwsSecretKey(config, "secret-key"); + + Items = TItem::Parse(config, "item"); + if (Items.empty()) { + throw TMissUseException() << "At least one item should be provided"; + } + + for (auto& item : Items) { + NConsoleClient::AdjustPath(item.Source, config); + } +} + +int TCommandExportToS3::Run(TConfig& config) { + using namespace NExport; + using namespace NScheme; + + TExportToS3Settings settings = FillSettings(TExportToS3Settings()); + + settings.Endpoint(AwsEndpoint); + settings.Scheme(AwsScheme); + settings.StorageClass(AwsStorageClass); + settings.Bucket(AwsBucket); + settings.AccessKey(AwsAccessKey); + settings.SecretKey(AwsSecretKey); + + for (const auto& item : Items) { + settings.AppendItem({item.Source, item.Destination}); + } + + if (Description) { + settings.Description(Description); + } + + settings.NumberOfRetries(NumberOfRetries); + + const TDriver driver = CreateDriver(config); + + TSchemeClient schemeClient(driver); + ExpandItems(schemeClient, settings, ExclusionPatterns); + + TExportClient client(driver); + TExportToS3Response response = client.ExportToS3(std::move(settings)).GetValueSync(); + ThrowOnError(response); + PrintOperation(response, OutputFormat); + + return EXIT_SUCCESS; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h index 0a12f12a8b5..6f842321ecf 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h @@ -1,76 +1,76 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/ydb_export/export.h> #include <ydb/public/lib/ydb_cli/common/aws.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/parseable_struct.h> #include <ydb/public/lib/ydb_cli/common/yt.h> - -#include <library/cpp/regex/pcre/regexp.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandExport : public TClientCommandTree { -public: + +#include <library/cpp/regex/pcre/regexp.h> + +namespace NYdb { +namespace NConsoleClient { + +class TCommandExport : public TClientCommandTree { +public: TCommandExport(bool useExportToYt); -}; - -class TCommandExportToYt : public TYdbOperationCommand, - public TCommandWithYtProxy, - public TCommandWithYtToken, - public TCommandWithFormat { -public: - TCommandExportToYt(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - struct TItemFields { - TString Source; - TString Destination; - bool Append = true; - }; - DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination, Append); - - TVector<TItem> Items; - TVector<TRegExMatch> ExclusionPatterns; - TString Description; - ui32 NumberOfRetries = 1000; - bool UseTypeV3 = false; -}; - -class TCommandExportToS3 : public TYdbOperationCommand, - public TCommandWithAwsCredentials, - public TCommandWithFormat { - using EStorageClass = NExport::TExportToS3Settings::EStorageClass; - -public: - TCommandExportToS3(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - struct TItemFields { - TString Source; - TString Destination; - }; - DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); - - TString AwsEndpoint; - ES3Scheme AwsScheme = ES3Scheme::HTTPS; - EStorageClass AwsStorageClass = EStorageClass::NOT_SET; - TString AwsBucket; - TVector<TItem> Items; - TVector<TRegExMatch> ExclusionPatterns; - TString Description; - ui32 NumberOfRetries = 10; -}; - -} -} +}; + +class TCommandExportToYt : public TYdbOperationCommand, + public TCommandWithYtProxy, + public TCommandWithYtToken, + public TCommandWithFormat { +public: + TCommandExportToYt(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + struct TItemFields { + TString Source; + TString Destination; + bool Append = true; + }; + DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination, Append); + + TVector<TItem> Items; + TVector<TRegExMatch> ExclusionPatterns; + TString Description; + ui32 NumberOfRetries = 1000; + bool UseTypeV3 = false; +}; + +class TCommandExportToS3 : public TYdbOperationCommand, + public TCommandWithAwsCredentials, + public TCommandWithFormat { + using EStorageClass = NExport::TExportToS3Settings::EStorageClass; + +public: + TCommandExportToS3(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + struct TItemFields { + TString Source; + TString Destination; + }; + DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); + + TString AwsEndpoint; + ES3Scheme AwsScheme = ES3Scheme::HTTPS; + EStorageClass AwsStorageClass = EStorageClass::NOT_SET; + TString AwsBucket; + TVector<TItem> Items; + TVector<TRegExMatch> ExclusionPatterns; + TString Description; + ui32 NumberOfRetries = 10; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp index 2f061a64e69..b28e8814288 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp @@ -1,128 +1,128 @@ -#include "ydb_service_import.h" - +#include "ydb_service_import.h" + #include <ydb/public/lib/ydb_cli/common/normalize_path.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> #include <ydb/public/lib/ydb_cli/import/import.h> #include <ydb/library/backup/util.h> - -#include <util/string/builder.h> -#include <util/string/join.h> + +#include <util/string/builder.h> +#include <util/string/join.h> #include <util/stream/format.h> // for SF_BYTES - -namespace NYdb { -namespace NConsoleClient { - -TCommandImport::TCommandImport() - : TClientCommandTree("import", {}, "Import service operations") -{ + +namespace NYdb { +namespace NConsoleClient { + +TCommandImport::TCommandImport() + : TClientCommandTree("import", {}, "Import service operations") +{ AddCommand(std::make_unique<TCommandImportFromS3>()); AddCommand(std::make_unique<TCommandImportFromFile>()); -} - -/// S3 -TCommandImportFromS3::TCommandImportFromS3() - : TYdbOperationCommand("s3", {}, "Create import from S3") -{ - TItem::DefineFields({ - {"Source", {{"source", "src", "s"}, "S3 object key prefix", true}}, - {"Destination", {{"destination", "dst", "d"}, "Database path to a table to import to", true}}, - }); -} - -void TCommandImportFromS3::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.Opts->AddLongOption("s3-endpoint", "S3 endpoint to connect to") - .Required().RequiredArgument("ENDPOINT").StoreResult(&AwsEndpoint); - - config.Opts->AddLongOption("scheme", "S3 endpoint scheme") - .RequiredArgument("SCHEME").StoreResult(&AwsScheme).DefaultValue(AwsScheme); - - config.Opts->AddLongOption("bucket", "S3 bucket") - .Required().RequiredArgument("BUCKET").StoreResult(&AwsBucket); - - TStringBuilder accessKeyHelp; - accessKeyHelp << "Access key id" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"AWS_ACCESS_KEY_ID\" environment variable" << Endl - << " 3. \"aws_access_key_id\" key in \"" << AwsCredentialsFile << "\" file"; - config.Opts->AddLongOption("access-key", accessKeyHelp) - .RequiredArgument("STRING"); - - TStringBuilder secretKeyHelp; - secretKeyHelp << "Secret key" << Endl - << " Search order:" << Endl - << " 1. This option" << Endl - << " 2. \"AWS_SECRET_ACCESS_KEY\" environment variable" << Endl - << " 3. \"aws_secret_access_key\" key in \"" << AwsCredentialsFile << "\" file"; - config.Opts->AddLongOption("secret-key", secretKeyHelp) - .RequiredArgument("STRING"); - - TStringBuilder itemHelp; - itemHelp << "[At least one] Item specification" << Endl - << " Possible property names:" << Endl - << TItem::FormatHelp(2); - config.Opts->AddLongOption("item", itemHelp) - .RequiredArgument("PROPERTY=VALUE,..."); - - config.Opts->AddLongOption("description", "Textual description of import operation") - .RequiredArgument("STRING").StoreResult(&Description); - - config.Opts->AddLongOption("retries", "Number of retries") - .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); - - AddJsonOption(config); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); -} - -void TCommandImportFromS3::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParseFormats(); - - ParseAwsAccessKey(config, "access-key"); - ParseAwsSecretKey(config, "secret-key"); - - Items = TItem::Parse(config, "item"); - if (Items.empty()) { - throw TMissUseException() << "At least one item should be provided"; - } - - for (auto& item : Items) { - NConsoleClient::AdjustPath(item.Destination, config); - } -} - -int TCommandImportFromS3::Run(TConfig& config) { - using namespace NImport; - - TImportFromS3Settings settings = FillSettings(TImportFromS3Settings()); - - settings.Endpoint(AwsEndpoint); - settings.Scheme(AwsScheme); - settings.Bucket(AwsBucket); - settings.AccessKey(AwsAccessKey); - settings.SecretKey(AwsSecretKey); - - for (const auto& item : Items) { - settings.AppendItem({item.Source, item.Destination}); - } - - if (Description) { - settings.Description(Description); - } - - settings.NumberOfRetries(NumberOfRetries); - - TImportClient client(CreateDriver(config)); - TImportFromS3Response response = client.ImportFromS3(std::move(settings)).GetValueSync(); - ThrowOnError(response); - PrintOperation(response, OutputFormat); - - return EXIT_SUCCESS; -} - +} + +/// S3 +TCommandImportFromS3::TCommandImportFromS3() + : TYdbOperationCommand("s3", {}, "Create import from S3") +{ + TItem::DefineFields({ + {"Source", {{"source", "src", "s"}, "S3 object key prefix", true}}, + {"Destination", {{"destination", "dst", "d"}, "Database path to a table to import to", true}}, + }); +} + +void TCommandImportFromS3::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.Opts->AddLongOption("s3-endpoint", "S3 endpoint to connect to") + .Required().RequiredArgument("ENDPOINT").StoreResult(&AwsEndpoint); + + config.Opts->AddLongOption("scheme", "S3 endpoint scheme") + .RequiredArgument("SCHEME").StoreResult(&AwsScheme).DefaultValue(AwsScheme); + + config.Opts->AddLongOption("bucket", "S3 bucket") + .Required().RequiredArgument("BUCKET").StoreResult(&AwsBucket); + + TStringBuilder accessKeyHelp; + accessKeyHelp << "Access key id" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"AWS_ACCESS_KEY_ID\" environment variable" << Endl + << " 3. \"aws_access_key_id\" key in \"" << AwsCredentialsFile << "\" file"; + config.Opts->AddLongOption("access-key", accessKeyHelp) + .RequiredArgument("STRING"); + + TStringBuilder secretKeyHelp; + secretKeyHelp << "Secret key" << Endl + << " Search order:" << Endl + << " 1. This option" << Endl + << " 2. \"AWS_SECRET_ACCESS_KEY\" environment variable" << Endl + << " 3. \"aws_secret_access_key\" key in \"" << AwsCredentialsFile << "\" file"; + config.Opts->AddLongOption("secret-key", secretKeyHelp) + .RequiredArgument("STRING"); + + TStringBuilder itemHelp; + itemHelp << "[At least one] Item specification" << Endl + << " Possible property names:" << Endl + << TItem::FormatHelp(2); + config.Opts->AddLongOption("item", itemHelp) + .RequiredArgument("PROPERTY=VALUE,..."); + + config.Opts->AddLongOption("description", "Textual description of import operation") + .RequiredArgument("STRING").StoreResult(&Description); + + config.Opts->AddLongOption("retries", "Number of retries") + .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); + + AddJsonOption(config); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); +} + +void TCommandImportFromS3::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParseFormats(); + + ParseAwsAccessKey(config, "access-key"); + ParseAwsSecretKey(config, "secret-key"); + + Items = TItem::Parse(config, "item"); + if (Items.empty()) { + throw TMissUseException() << "At least one item should be provided"; + } + + for (auto& item : Items) { + NConsoleClient::AdjustPath(item.Destination, config); + } +} + +int TCommandImportFromS3::Run(TConfig& config) { + using namespace NImport; + + TImportFromS3Settings settings = FillSettings(TImportFromS3Settings()); + + settings.Endpoint(AwsEndpoint); + settings.Scheme(AwsScheme); + settings.Bucket(AwsBucket); + settings.AccessKey(AwsAccessKey); + settings.SecretKey(AwsSecretKey); + + for (const auto& item : Items) { + settings.AppendItem({item.Source, item.Destination}); + } + + if (Description) { + settings.Description(Description); + } + + settings.NumberOfRetries(NumberOfRetries); + + TImportClient client(CreateDriver(config)); + TImportFromS3Response response = client.ImportFromS3(std::move(settings)).GetValueSync(); + ThrowOnError(response); + PrintOperation(response, OutputFormat); + + return EXIT_SUCCESS; +} + /// File TCommandImportFromFile::TCommandImportFromFile() @@ -130,7 +130,7 @@ TCommandImportFromFile::TCommandImportFromFile() { AddCommand(std::make_unique<TCommandImportFromCsv>()); AddCommand(std::make_unique<TCommandImportFromTsv>()); -} +} /// CSV @@ -170,7 +170,7 @@ void TCommandImportFromCsv::Config(TConfig& config) { config.Opts->AddLongOption("max-in-flight", "Maximum number of in-flight requests; increase to load big files faster (more memory needed)") .DefaultValue(defaults.MaxInFlightRequests_).StoreResult(&MaxInFlightRequests); -} +} void TCommandImportFromCsv::Parse(TConfig& config) { TClientCommand::Parse(config); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h index 7321d7c967f..04dc1010476 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h @@ -1,46 +1,46 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/lib/ydb_cli/common/aws.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/parseable_struct.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandImport : public TClientCommandTree { -public: - TCommandImport(); -}; - -class TCommandImportFromS3 : public TYdbOperationCommand, - public TCommandWithAwsCredentials, - public TCommandWithFormat { -public: - TCommandImportFromS3(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - struct TItemFields { - TString Source; - TString Destination; - }; - DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); - - TString AwsEndpoint; - ES3Scheme AwsScheme = ES3Scheme::HTTPS; - TString AwsBucket; - TVector<TItem> Items; - TString Description; - ui32 NumberOfRetries = 10; -}; - + +namespace NYdb { +namespace NConsoleClient { + +class TCommandImport : public TClientCommandTree { +public: + TCommandImport(); +}; + +class TCommandImportFromS3 : public TYdbOperationCommand, + public TCommandWithAwsCredentials, + public TCommandWithFormat { +public: + TCommandImportFromS3(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + struct TItemFields { + TString Source; + TString Destination; + }; + DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); + + TString AwsEndpoint; + ES3Scheme AwsScheme = ES3Scheme::HTTPS; + TString AwsBucket; + TVector<TItem> Items; + TString Description; + ui32 NumberOfRetries = 10; +}; + class TCommandImportFromFile : public TClientCommandTree { public: TCommandImportFromFile(); @@ -75,5 +75,5 @@ public: } }; -} -} +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp index 119d1c17243..45b694cf948 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp @@ -1,134 +1,134 @@ -#include "ydb_service_operation.h" - +#include "ydb_service_operation.h" + #include <ydb/public/sdk/cpp/client/ydb_export/export.h> #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> - -#include <util/string/builder.h> - -namespace NYdb { -namespace NConsoleClient { - -using namespace NKikimr::NOperationId; - -namespace { - - template <typename T> - int GetOperation(NOperation::TOperationClient& client, const TOperationId& id, EOutputFormat format) { - T operation = client.Get<T>(id).GetValueSync(); - switch (operation.Status().GetStatus()) { - case EStatus::SUCCESS: - PrintOperation(operation, format); - return EXIT_SUCCESS; - case EStatus::CANCELLED: - PrintOperation(operation, format); - return EXIT_FAILURE; - default: - ThrowOnError(operation); - return EXIT_FAILURE; - } - } - - template <typename T> - void ListOperations(NOperation::TOperationClient& client, ui64 pageSize, const TString& pageToken, EOutputFormat format) { - NOperation::TOperationsList<T> operations = client.List<T>(pageSize, pageToken).GetValueSync(); - ThrowOnError(operations); - PrintOperationsList(operations, format); - } - -} // anonymous - -TCommandOperation::TCommandOperation() - : TClientCommandTree("operation", {}, "Operation service operations") -{ + +#include <util/string/builder.h> + +namespace NYdb { +namespace NConsoleClient { + +using namespace NKikimr::NOperationId; + +namespace { + + template <typename T> + int GetOperation(NOperation::TOperationClient& client, const TOperationId& id, EOutputFormat format) { + T operation = client.Get<T>(id).GetValueSync(); + switch (operation.Status().GetStatus()) { + case EStatus::SUCCESS: + PrintOperation(operation, format); + return EXIT_SUCCESS; + case EStatus::CANCELLED: + PrintOperation(operation, format); + return EXIT_FAILURE; + default: + ThrowOnError(operation); + return EXIT_FAILURE; + } + } + + template <typename T> + void ListOperations(NOperation::TOperationClient& client, ui64 pageSize, const TString& pageToken, EOutputFormat format) { + NOperation::TOperationsList<T> operations = client.List<T>(pageSize, pageToken).GetValueSync(); + ThrowOnError(operations); + PrintOperationsList(operations, format); + } + +} // anonymous + +TCommandOperation::TCommandOperation() + : TClientCommandTree("operation", {}, "Operation service operations") +{ AddCommand(std::make_unique<TCommandGetOperation>()); AddCommand(std::make_unique<TCommandCancelOperation>()); AddCommand(std::make_unique<TCommandForgetOperation>()); AddCommand(std::make_unique<TCommandListOperations>()); -} - -void TCommandWithOperationId::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<id>", "Operation ID"); -} - -void TCommandWithOperationId::Parse(TConfig& config) { - TYdbCommand::Parse(config); - - try { - OperationId = TOperationId(config.ParseResult->GetFreeArgs()[0]); - } catch (const yexception& ex) { - throw TMissUseException() << "Invalid operation ID"; - } -} - -TCommandGetOperation::TCommandGetOperation() - : TCommandWithOperationId("get", {}, "Check status for a given operation") -{ -} - -void TCommandGetOperation::Config(TConfig& config) { - TCommandWithOperationId::Config(config); - AddJsonOption(config); +} + +void TCommandWithOperationId::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<id>", "Operation ID"); +} + +void TCommandWithOperationId::Parse(TConfig& config) { + TYdbCommand::Parse(config); + + try { + OperationId = TOperationId(config.ParseResult->GetFreeArgs()[0]); + } catch (const yexception& ex) { + throw TMissUseException() << "Invalid operation ID"; + } +} + +TCommandGetOperation::TCommandGetOperation() + : TCommandWithOperationId("get", {}, "Check status for a given operation") +{ +} + +void TCommandGetOperation::Config(TConfig& config) { + TCommandWithOperationId::Config(config); + AddJsonOption(config); AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); config.Opts->MutuallyExclusive("json", "format"); -} - +} + void TCommandGetOperation::Parse(TConfig& config) { TCommandWithOperationId::Parse(config); ParseFormats(); } -int TCommandGetOperation::Run(TConfig& config) { - NOperation::TOperationClient client(CreateDriver(config)); - - switch (OperationId.GetKind()) { - case Ydb::TOperationId::EXPORT: - if (OperationId.GetSubKind() == "s3") { - return GetOperation<NExport::TExportToS3Response>(client, OperationId, OutputFormat); - } else { // fallback to "yt" - return GetOperation<NExport::TExportToYtResponse>(client, OperationId, OutputFormat); - } - case Ydb::TOperationId::IMPORT: - if (OperationId.GetSubKind() == "s3") { - return GetOperation<NImport::TImportFromS3Response>(client, OperationId, OutputFormat); - } else { - throw TMissUseException() << "Invalid operation ID (unexpected sub-kind of operation)"; - } +int TCommandGetOperation::Run(TConfig& config) { + NOperation::TOperationClient client(CreateDriver(config)); + + switch (OperationId.GetKind()) { + case Ydb::TOperationId::EXPORT: + if (OperationId.GetSubKind() == "s3") { + return GetOperation<NExport::TExportToS3Response>(client, OperationId, OutputFormat); + } else { // fallback to "yt" + return GetOperation<NExport::TExportToYtResponse>(client, OperationId, OutputFormat); + } + case Ydb::TOperationId::IMPORT: + if (OperationId.GetSubKind() == "s3") { + return GetOperation<NImport::TImportFromS3Response>(client, OperationId, OutputFormat); + } else { + throw TMissUseException() << "Invalid operation ID (unexpected sub-kind of operation)"; + } case Ydb::TOperationId::BUILD_INDEX: - return GetOperation<NTable::TBuildIndexOperation>(client, OperationId, OutputFormat); - default: + return GetOperation<NTable::TBuildIndexOperation>(client, OperationId, OutputFormat); + default: throw TMissUseException() << "Invalid operation ID (unexpected kind of operation)"; - } - - return EXIT_SUCCESS; -} - -TCommandCancelOperation::TCommandCancelOperation() - : TCommandWithOperationId("cancel", {}, "Start cancellation of a long-running operation") -{ -} - -int TCommandCancelOperation::Run(TConfig& config) { - NOperation::TOperationClient client(CreateDriver(config)); - ThrowOnError(client.Cancel(OperationId).GetValueSync()); - return EXIT_SUCCESS; -} - -TCommandForgetOperation::TCommandForgetOperation() - : TCommandWithOperationId("forget", {}, "Forget long-running operation") -{ -} - -int TCommandForgetOperation::Run(TConfig& config) { - NOperation::TOperationClient client(CreateDriver(config)); - ThrowOnError(client.Forget(OperationId).GetValueSync()); - return EXIT_SUCCESS; -} - + } + + return EXIT_SUCCESS; +} + +TCommandCancelOperation::TCommandCancelOperation() + : TCommandWithOperationId("cancel", {}, "Start cancellation of a long-running operation") +{ +} + +int TCommandCancelOperation::Run(TConfig& config) { + NOperation::TOperationClient client(CreateDriver(config)); + ThrowOnError(client.Cancel(OperationId).GetValueSync()); + return EXIT_SUCCESS; +} + +TCommandForgetOperation::TCommandForgetOperation() + : TCommandWithOperationId("forget", {}, "Forget long-running operation") +{ +} + +int TCommandForgetOperation::Run(TConfig& config) { + NOperation::TOperationClient client(CreateDriver(config)); + ThrowOnError(client.Forget(OperationId).GetValueSync()); + return EXIT_SUCCESS; +} + void TCommandListOperations::InitializeKindToHandler(TConfig& config) { KindToHandler = { {"export/s3", &ListOperations<NExport::TExportToS3Response>}, @@ -140,59 +140,59 @@ void TCommandListOperations::InitializeKindToHandler(TConfig& config) { KindToHandler.emplace("export/yt", &ListOperations<NExport::TExportToYtResponse>); } } - -TString TCommandListOperations::KindChoices() { - TStringBuilder help; - - bool first = true; - for (const auto& kv : KindToHandler) { - if (!first) { - help << ", "; - } - help << kv.first; - first = false; - } - - return help; -} - -TCommandListOperations::TCommandListOperations() + +TString TCommandListOperations::KindChoices() { + TStringBuilder help; + + bool first = true; + for (const auto& kv : KindToHandler) { + if (!first) { + help << ", "; + } + help << kv.first; + first = false; + } + + return help; +} + +TCommandListOperations::TCommandListOperations() : TYdbCommand("list", {}, "List operations of specified kind") -{ -} - -void TCommandListOperations::Config(TConfig& config) { - TYdbCommand::Config(config); - +{ +} + +void TCommandListOperations::Config(TConfig& config) { + TYdbCommand::Config(config); + InitializeKindToHandler(config); - config.Opts->AddLongOption('s', "page-size", "Page size") - .RequiredArgument("NUM").StoreResult(&PageSize); - config.Opts->AddLongOption('t', "page-token", "Page token") - .RequiredArgument("STRING").StoreResult(&PageToken); - AddJsonOption(config); + config.Opts->AddLongOption('s', "page-size", "Page size") + .RequiredArgument("NUM").StoreResult(&PageSize); + config.Opts->AddLongOption('t', "page-token", "Page token") + .RequiredArgument("STRING").StoreResult(&PageToken); + AddJsonOption(config); AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); config.Opts->MutuallyExclusive("json", "format"); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<kind>", KindChoices()); -} - -void TCommandListOperations::Parse(TConfig& config) { - TYdbCommand::Parse(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<kind>", KindChoices()); +} + +void TCommandListOperations::Parse(TConfig& config) { + TYdbCommand::Parse(config); ParseFormats(); - - Kind = config.ParseResult->GetFreeArgs()[0]; - if (!KindToHandler.contains(Kind)) { - throw TMissUseException() << "Invalid kind. Use one of: " << KindChoices(); - } -} - -int TCommandListOperations::Run(TConfig& config) { - NOperation::TOperationClient client(CreateDriver(config)); - KindToHandler.at(Kind)(client, PageSize, PageToken, OutputFormat); - return EXIT_SUCCESS; -} - -} -} + + Kind = config.ParseResult->GetFreeArgs()[0]; + if (!KindToHandler.contains(Kind)) { + throw TMissUseException() << "Invalid kind. Use one of: " << KindChoices(); + } +} + +int TCommandListOperations::Run(TConfig& config) { + NOperation::TOperationClient client(CreateDriver(config)); + KindToHandler.at(Kind)(client, PageSize, PageToken, OutputFormat); + return EXIT_SUCCESS; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h index b708b426f6f..238901a88b8 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h @@ -1,73 +1,73 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/lib/operation_id/operation_id.h> #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> #include <ydb/public/lib/ydb_cli/common/format.h> - -#include <util/generic/hash.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandOperation : public TClientCommandTree { -public: - TCommandOperation(); -}; - -class TCommandWithOperationId : public TYdbCommand { -public: - using TYdbCommand::TYdbCommand; - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - -protected: - NKikimr::NOperationId::TOperationId OperationId; -}; - -class TCommandGetOperation : public TCommandWithOperationId, - public TCommandWithFormat { -public: - TCommandGetOperation(); - virtual void Config(TConfig& config) override; + +#include <util/generic/hash.h> + +namespace NYdb { +namespace NConsoleClient { + +class TCommandOperation : public TClientCommandTree { +public: + TCommandOperation(); +}; + +class TCommandWithOperationId : public TYdbCommand { +public: + using TYdbCommand::TYdbCommand; + virtual void Config(TConfig& config) override; virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -class TCommandCancelOperation : public TCommandWithOperationId { -public: - TCommandCancelOperation(); - virtual int Run(TConfig& config) override; -}; - -class TCommandForgetOperation : public TCommandWithOperationId { -public: - TCommandForgetOperation(); - virtual int Run(TConfig& config) override; -}; - -class TCommandListOperations : public TYdbCommand, - public TCommandWithFormat { - - using THandler = std::function<void(NOperation::TOperationClient&, ui64, const TString&, EOutputFormat)>; - + +protected: + NKikimr::NOperationId::TOperationId OperationId; +}; + +class TCommandGetOperation : public TCommandWithOperationId, + public TCommandWithFormat { +public: + TCommandGetOperation(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +class TCommandCancelOperation : public TCommandWithOperationId { +public: + TCommandCancelOperation(); + virtual int Run(TConfig& config) override; +}; + +class TCommandForgetOperation : public TCommandWithOperationId { +public: + TCommandForgetOperation(); + virtual int Run(TConfig& config) override; +}; + +class TCommandListOperations : public TYdbCommand, + public TCommandWithFormat { + + using THandler = std::function<void(NOperation::TOperationClient&, ui64, const TString&, EOutputFormat)>; + void InitializeKindToHandler(TConfig& config); TString KindChoices(); - -public: - TCommandListOperations(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString Kind; - ui64 PageSize = 0; - TString PageToken; + +public: + TCommandListOperations(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString Kind; + ui64 PageSize = 0; + TString PageToken; THashMap<TString, THandler> KindToHandler; -}; - -} -} +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index f13d9004580..a6f81fe432a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -452,29 +452,29 @@ namespace { if (!settings) { return; } - Cout << Endl << "Ttl settings "; + Cout << Endl << "Ttl settings "; switch (settings->GetMode()) { case NTable::TTtlSettings::EMode::DateTypeColumn: { - Cout << "(date type column):" << Endl; + Cout << "(date type column):" << Endl; const auto& dateTypeColumn = settings->GetDateTypeColumn(); Cout << "Column name: " << dateTypeColumn.GetColumnName() << Endl; Cout << "Expire after: " << dateTypeColumn.GetExpireAfter() << Endl; break; } - case NTable::TTtlSettings::EMode::ValueSinceUnixEpoch: - { - Cout << "(value since unix epoch):" << Endl; - const auto& valueSinceEpoch = settings->GetValueSinceUnixEpoch(); - Cout << "Column name: " << valueSinceEpoch.GetColumnName() << Endl; - Cout << "Column unit: " << valueSinceEpoch.GetColumnUnit() << Endl; - Cout << "Expire after: " << valueSinceEpoch.GetExpireAfter() << Endl; - break; - } + case NTable::TTtlSettings::EMode::ValueSinceUnixEpoch: + { + Cout << "(value since unix epoch):" << Endl; + const auto& valueSinceEpoch = settings->GetValueSinceUnixEpoch(); + Cout << "Column name: " << valueSinceEpoch.GetColumnName() << Endl; + Cout << "Column unit: " << valueSinceEpoch.GetColumnUnit() << Endl; + Cout << "Expire after: " << valueSinceEpoch.GetExpireAfter() << Endl; + break; + } default: NColorizer::TColors colors = NColorizer::AutoColors(Cout); - Cout << "(unknown):" << Endl - << colors.RedColor() << "Unknown ttl settings mode. Please update your version of YDB cli" + Cout << "(unknown):" << Endl + << colors.RedColor() << "Unknown ttl settings mode. Please update your version of YDB cli" << colors.OldColor() << Endl; } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 8bd037fb97c..2f94b2fc033 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -44,20 +44,20 @@ TCommandIndex::TCommandIndex() AddCommand(std::make_unique<TCommandIndexDrop>()); } -TCommandAttribute::TCommandAttribute() - : TClientCommandTree("attribute", {"attr"}, "Attribute operations") -{ +TCommandAttribute::TCommandAttribute() + : TClientCommandTree("attribute", {"attr"}, "Attribute operations") +{ AddCommand(std::make_unique<TCommandAttributeAdd>()); AddCommand(std::make_unique<TCommandAttributeDrop>()); -} - -TCommandTtl::TCommandTtl() - : TClientCommandTree("ttl", {}, "Ttl operations") -{ +} + +TCommandTtl::TCommandTtl() + : TClientCommandTree("ttl", {}, "Ttl operations") +{ AddCommand(std::make_unique<TCommandTtlSet>()); AddCommand(std::make_unique<TCommandTtlDrop>()); -} - +} + TCommandIndexAdd::TCommandIndexAdd() : TClientCommandTree("add", {}, "Add index in to the specified table") { @@ -838,13 +838,13 @@ void TCommandReadTable::PrintResponse(NTable::TTablePartIterator& result) { } } -TCommandIndexAddGlobal::TCommandIndexAddGlobal( - NTable::EIndexType type, - const TString& name, - const std::initializer_list<TString>& aliases, - const TString& description) - : TYdbCommand(name, aliases, description) - , IndexType(type) +TCommandIndexAddGlobal::TCommandIndexAddGlobal( + NTable::EIndexType type, + const TString& name, + const std::initializer_list<TString>& aliases, + const TString& description) + : TYdbCommand(name, aliases, description) + , IndexType(type) {} void TCommandIndexAddGlobal::Config(TConfig& config) { @@ -886,14 +886,14 @@ int TCommandIndexAddGlobal::Run(TConfig& config) { return EXIT_SUCCESS; } -TCommandIndexAddGlobalSync::TCommandIndexAddGlobalSync() - : TCommandIndexAddGlobal(NTable::EIndexType::GlobalSync, "global-sync", {"global"}, "Add global sync index. The command returns operation") -{} - -TCommandIndexAddGlobalAsync::TCommandIndexAddGlobalAsync() - : TCommandIndexAddGlobal(NTable::EIndexType::GlobalAsync, "global-async", {}, "Add global async index. The command returns operation") -{} - +TCommandIndexAddGlobalSync::TCommandIndexAddGlobalSync() + : TCommandIndexAddGlobal(NTable::EIndexType::GlobalSync, "global-sync", {"global"}, "Add global sync index. The command returns operation") +{} + +TCommandIndexAddGlobalAsync::TCommandIndexAddGlobalAsync() + : TCommandIndexAddGlobal(NTable::EIndexType::GlobalAsync, "global-async", {}, "Add global async index. The command returns operation") +{} + TCommandIndexDrop::TCommandIndexDrop() : TYdbCommand("drop", {}, "Drop index from the specified table") {} @@ -926,180 +926,180 @@ int TCommandIndexDrop::Run(TConfig& config) { return EXIT_SUCCESS; } -TCommandAttributeAdd::TCommandAttributeAdd() - : TYdbCommand("add", {}, "Add attributes to the specified table") -{} - -void TCommandAttributeAdd::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("attribute", "key=value pair to add.") - .RequiredArgument("KEY=VALUE").KVHandler([&](TString key, TString value) { - Attributes[key] = value; - }); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); +TCommandAttributeAdd::TCommandAttributeAdd() + : TYdbCommand("add", {}, "Add attributes to the specified table") +{} + +void TCommandAttributeAdd::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("attribute", "key=value pair to add.") + .RequiredArgument("KEY=VALUE").KVHandler([&](TString key, TString value) { + Attributes[key] = value; + }); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); +} + +void TCommandAttributeAdd::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandAttributeAdd::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + auto settings = NTable::TAlterTableSettings() + .AlterAttributes(Attributes); + + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); + ThrowOnError(result); + + return EXIT_SUCCESS; +} + +TCommandAttributeDrop::TCommandAttributeDrop() + : TYdbCommand("drop", {}, "Drop attributes from the specified table") +{} + +void TCommandAttributeDrop::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("attributes", "Attribute keys to drop.") + .RequiredArgument("KEY,[KEY...]").SplitHandler(&AttributeKeys, ','); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); +} + +void TCommandAttributeDrop::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandAttributeDrop::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + auto settings = NTable::TAlterTableSettings(); + auto alterAttrs = settings.BeginAlterAttributes(); + for (const auto& key : AttributeKeys) { + alterAttrs.Drop(key); + } + alterAttrs.EndAlterAttributes(); + + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); + ThrowOnError(result); + + return EXIT_SUCCESS; +} + +TCommandTtlSet::TCommandTtlSet() + : TYdbCommand("set", {}, "Set ttl settings for the specified table") +{} + +void TCommandTtlSet::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("column", "Name of date- or integral-type column to be used to calculate expiration threshold.") + .RequiredArgument("NAME").StoreResult(&ColumnName); + config.Opts->AddLongOption("expire-after", "Additional time that must pass since expiration threshold.") + .RequiredArgument("SECONDS").DefaultValue(0).Handler1T<TDuration::TValue>(0, [this](const TDuration::TValue& arg) { + ExpireAfter = TDuration::Seconds(arg); + }); + + const TString allowedUnits = "seconds (s, sec), milliseconds (ms, msec), microseconds (us, usec), nanoseconds (ns, nsec)"; + auto unitHelp = TStringBuilder() + << "Interpretation of the value stored in integral-type column." << Endl + << "Allowed units: " << allowedUnits; + config.Opts->AddLongOption("unit", unitHelp) + .RequiredArgument("STRING").Handler1T<TString>("", [this, allowedUnits](const TString& arg) { + if (!arg) { + return; + } + + const auto value = NTable::TValueSinceUnixEpochModeSettings::UnitFromString(arg); + if (value == NTable::TTtlSettings::EUnit::Unknown) { + throw TMissUseException() << "Unknown unit: " << arg << Endl << "Allowed units: " << allowedUnits; + } + + ColumnUnit = value; + }); + + config.Opts->AddLongOption("run-interval", "[Advanced] How often to run cleanup operation on the same partition.") + .RequiredArgument("SECONDS").Handler1T<TDuration::TValue>([this](const TDuration::TValue& arg) { + RunInterval = TDuration::Seconds(arg); + }); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); +} + +void TCommandTtlSet::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandTtlSet::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + TMaybe<NTable::TTtlSettings> ttl; + if (ColumnUnit) { + ttl = NTable::TTtlSettings(ColumnName, *ColumnUnit, ExpireAfter); + } else { + ttl = NTable::TTtlSettings(ColumnName, ExpireAfter); + } + if (RunInterval) { + ttl->SetRunInterval(RunInterval); + } + + auto settings = NTable::TAlterTableSettings() + .BeginAlterTtlSettings() + .Set(std::move(ttl.GetRef())) + .EndAlterTtlSettings(); + + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); + ThrowOnError(result); + + return EXIT_SUCCESS; +} + +TCommandTtlDrop::TCommandTtlDrop() + : TYdbCommand("drop", {}, "Drop ttl settings from the specified table") +{} + +void TCommandTtlDrop::Config(TConfig& config) { + TYdbCommand::Config(config); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); } - -void TCommandAttributeAdd::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); + +void TCommandTtlDrop::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); } -int TCommandAttributeAdd::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - auto settings = NTable::TAlterTableSettings() - .AlterAttributes(Attributes); - - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); - - return EXIT_SUCCESS; -} - -TCommandAttributeDrop::TCommandAttributeDrop() - : TYdbCommand("drop", {}, "Drop attributes from the specified table") -{} - -void TCommandAttributeDrop::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("attributes", "Attribute keys to drop.") - .RequiredArgument("KEY,[KEY...]").SplitHandler(&AttributeKeys, ','); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); -} - -void TCommandAttributeDrop::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandAttributeDrop::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - auto settings = NTable::TAlterTableSettings(); - auto alterAttrs = settings.BeginAlterAttributes(); - for (const auto& key : AttributeKeys) { - alterAttrs.Drop(key); - } - alterAttrs.EndAlterAttributes(); - - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); - - return EXIT_SUCCESS; -} - -TCommandTtlSet::TCommandTtlSet() - : TYdbCommand("set", {}, "Set ttl settings for the specified table") -{} - -void TCommandTtlSet::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("column", "Name of date- or integral-type column to be used to calculate expiration threshold.") - .RequiredArgument("NAME").StoreResult(&ColumnName); - config.Opts->AddLongOption("expire-after", "Additional time that must pass since expiration threshold.") - .RequiredArgument("SECONDS").DefaultValue(0).Handler1T<TDuration::TValue>(0, [this](const TDuration::TValue& arg) { - ExpireAfter = TDuration::Seconds(arg); - }); - - const TString allowedUnits = "seconds (s, sec), milliseconds (ms, msec), microseconds (us, usec), nanoseconds (ns, nsec)"; - auto unitHelp = TStringBuilder() - << "Interpretation of the value stored in integral-type column." << Endl - << "Allowed units: " << allowedUnits; - config.Opts->AddLongOption("unit", unitHelp) - .RequiredArgument("STRING").Handler1T<TString>("", [this, allowedUnits](const TString& arg) { - if (!arg) { - return; - } - - const auto value = NTable::TValueSinceUnixEpochModeSettings::UnitFromString(arg); - if (value == NTable::TTtlSettings::EUnit::Unknown) { - throw TMissUseException() << "Unknown unit: " << arg << Endl << "Allowed units: " << allowedUnits; - } - - ColumnUnit = value; - }); - - config.Opts->AddLongOption("run-interval", "[Advanced] How often to run cleanup operation on the same partition.") - .RequiredArgument("SECONDS").Handler1T<TDuration::TValue>([this](const TDuration::TValue& arg) { - RunInterval = TDuration::Seconds(arg); - }); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); -} - -void TCommandTtlSet::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandTtlSet::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - TMaybe<NTable::TTtlSettings> ttl; - if (ColumnUnit) { - ttl = NTable::TTtlSettings(ColumnName, *ColumnUnit, ExpireAfter); - } else { - ttl = NTable::TTtlSettings(ColumnName, ExpireAfter); - } - if (RunInterval) { - ttl->SetRunInterval(RunInterval); - } - - auto settings = NTable::TAlterTableSettings() - .BeginAlterTtlSettings() - .Set(std::move(ttl.GetRef())) - .EndAlterTtlSettings(); - - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); - - return EXIT_SUCCESS; -} - -TCommandTtlDrop::TCommandTtlDrop() - : TYdbCommand("drop", {}, "Drop ttl settings from the specified table") -{} - -void TCommandTtlDrop::Config(TConfig& config) { - TYdbCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); -} - -void TCommandTtlDrop::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandTtlDrop::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - auto settings = NTable::TAlterTableSettings() - .BeginAlterTtlSettings() - .Drop() - .EndAlterTtlSettings(); - - auto session = client.GetSession().GetValueSync(); - ThrowOnError(session); - auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); - ThrowOnError(result); - - return EXIT_SUCCESS; -} - -} -} +int TCommandTtlDrop::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + auto settings = NTable::TAlterTableSettings() + .BeginAlterTtlSettings() + .Drop() + .EndAlterTtlSettings(); + + auto session = client.GetSession().GetValueSync(); + ThrowOnError(session); + auto result = session.GetSession().AlterTable(Path, settings).GetValueSync(); + ThrowOnError(result); + + return EXIT_SUCCESS; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index 54da2ed7ec3..5cd2514aa02 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -28,16 +28,16 @@ public: TCommandIndex(); }; -class TCommandAttribute : public TClientCommandTree { -public: - TCommandAttribute(); -}; - -class TCommandTtl: public TClientCommandTree { -public: - TCommandTtl(); -}; - +class TCommandAttribute : public TClientCommandTree { +public: + TCommandAttribute(); +}; + +class TCommandTtl: public TClientCommandTree { +public: + TCommandTtl(); +}; + class TTableCommand : public TYdbOperationCommand { public: TTableCommand( @@ -158,33 +158,33 @@ public: class TCommandIndexAddGlobal : public TYdbCommand, public TCommandWithPath, public TCommandWithFormat { public: - TCommandIndexAddGlobal( - NTable::EIndexType type, - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ); + TCommandIndexAddGlobal( + NTable::EIndexType type, + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); virtual void Config(TConfig& config) override; virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; -protected: +protected: TString IndexName; TString Columns; TString DataColumns; -private: - const NTable::EIndexType IndexType; -}; - -class TCommandIndexAddGlobalSync : public TCommandIndexAddGlobal { -public: - TCommandIndexAddGlobalSync(); -}; - -class TCommandIndexAddGlobalAsync : public TCommandIndexAddGlobal { -public: - TCommandIndexAddGlobalAsync(); -}; - +private: + const NTable::EIndexType IndexType; +}; + +class TCommandIndexAddGlobalSync : public TCommandIndexAddGlobal { +public: + TCommandIndexAddGlobalSync(); +}; + +class TCommandIndexAddGlobalAsync : public TCommandIndexAddGlobal { +public: + TCommandIndexAddGlobalAsync(); +}; + class TCommandIndexDrop : public TYdbCommand, public TCommandWithPath { public: TCommandIndexDrop(); @@ -195,46 +195,46 @@ private: TString IndexName; }; -class TCommandAttributeAdd : public TYdbCommand, public TCommandWithPath { -public: - TCommandAttributeAdd(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -private: - THashMap<TString, TString> Attributes; -}; - -class TCommandAttributeDrop : public TYdbCommand, public TCommandWithPath { -public: - TCommandAttributeDrop(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -private: - TVector<TString> AttributeKeys; -}; - -class TCommandTtlSet : public TYdbCommand, public TCommandWithPath { -public: - TCommandTtlSet(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -private: - TString ColumnName; - TMaybe<NTable::TTtlSettings::EUnit> ColumnUnit; - TDuration ExpireAfter; - TDuration RunInterval = TDuration::Zero(); -}; - -class TCommandTtlDrop : public TYdbCommand, public TCommandWithPath { -public: - TCommandTtlDrop(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - +class TCommandAttributeAdd : public TYdbCommand, public TCommandWithPath { +public: + TCommandAttributeAdd(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +private: + THashMap<TString, TString> Attributes; +}; + +class TCommandAttributeDrop : public TYdbCommand, public TCommandWithPath { +public: + TCommandAttributeDrop(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +private: + TVector<TString> AttributeKeys; +}; + +class TCommandTtlSet : public TYdbCommand, public TCommandWithPath { +public: + TCommandTtlSet(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +private: + TString ColumnName; + TMaybe<NTable::TTtlSettings::EUnit> ColumnUnit; + TDuration ExpireAfter; + TDuration RunInterval = TDuration::Zero(); +}; + +class TCommandTtlDrop : public TYdbCommand, public TCommandWithPath { +public: + TCommandTtlDrop(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index e0c7095a9cf..db4ec14d30c 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -5,7 +5,7 @@ #include <ydb/library/backup/backup.h> #include <ydb/library/backup/util.h> -#include <util/stream/format.h> +#include <util/stream/format.h> #include <util/string/split.h> namespace NYdb::NConsoleClient { @@ -43,9 +43,9 @@ void TCommandDump::Config(TConfig& config) { .DefaultValue(".").StoreResult(&Path); config.Opts->AddLongOption("exclude", "Pattern(s) (PCRE) for paths excluded from dump." " Option can be used several times - one for each pattern.") - .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { - ExclusionPatterns.emplace_back(TRegExMatch(arg)); - }); + .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { + ExclusionPatterns.emplace_back(TRegExMatch(arg)); + }); config.Opts->AddLongOption('o', "output", "[Required] Path in a local filesystem to a directory to place dump into." " Directory should either not exist or be empty.") .StoreResult(&FilePath); @@ -54,7 +54,7 @@ void TCommandDump::Config(TConfig& config) { config.Opts->AddLongOption("avoid-copy", "Avoid copying." " By default, YDB makes a copy of a table before dumping it to reduce impact on workload and ensure consistency.\n" "In some cases (e.g. for tables with external blobs) copying should be disabled.") - .StoreTrue(&AvoidCopy); + .StoreTrue(&AvoidCopy); config.Opts->AddLongOption("save-partial-result", "Do not remove partial dump result." " If this option is not enabled, all files that have already been created will be removed in case of error.") .StoreTrue(&SavePartialResult); @@ -62,7 +62,7 @@ void TCommandDump::Config(TConfig& config) { " If this option is enabled, storage pool kind will be saved to dump." " In this case, if there will be no such storage pool kind in database on restore, error will occur." " By default this option is disabled and any existing storage pool kind will be used on restore.") - .StoreTrue(&PreservePoolKinds); + .StoreTrue(&PreservePoolKinds); config.Opts->AddLongOption("consistency-level", "Consistency level." " Options: database, table\n" "database - take one consistent snapshot of all tables specified for dump." @@ -91,8 +91,8 @@ int TCommandDump::Run(TConfig& config) { try { TString relPath = NYdb::RelPathFromAbsolute(config.Database, Path); - NYdb::NBackup::BackupFolder(CreateDriver(config), config.Database, relPath, FilePath, ExclusionPatterns, - IsSchemeOnly, useConsistentCopyTable, AvoidCopy, SavePartialResult, PreservePoolKinds); + NYdb::NBackup::BackupFolder(CreateDriver(config), config.Database, relPath, FilePath, ExclusionPatterns, + IsSchemeOnly, useConsistentCopyTable, AvoidCopy, SavePartialResult, PreservePoolKinds); } catch (const NYdb::NBackup::TYdbErrorException& e) { e.LogToStderr(); return EXIT_FAILURE; @@ -118,64 +118,64 @@ void TCommandRestore::Config(TConfig& config) { config.Opts->AddLongOption('p', "path", "[Required] Database path to a destination directory where restored directory or table will be placed.") .StoreResult(&Path); - config.Opts->AddLongOption('i', "input", + config.Opts->AddLongOption('i', "input", "[Required] Path in a local filesystem to a directory with dump.") .StoreResult(&FilePath); - config.Opts->AddLongOption("dry-run", TStringBuilder() - << "Do not restore tables, only check that:" << Endl + config.Opts->AddLongOption("dry-run", TStringBuilder() + << "Do not restore tables, only check that:" << Endl << " - all dumped tables exist in database;" << Endl << " - all dumped table schemes are the same as in database.") .StoreTrue(&IsDryRun); - - NDump::TRestoreSettings defaults; - - config.Opts->AddLongOption("restore-data", "Whether to restore data or not") - .DefaultValue(defaults.RestoreData_).StoreResult(&RestoreData); - - config.Opts->AddLongOption("restore-indexes", "Whether to restore indexes or not") - .DefaultValue(defaults.RestoreIndexes_).StoreResult(&RestoreIndexes); - - config.Opts->AddLongOption("skip-document-tables", TStringBuilder() - << "Document API tables cannot be restored for now. " - << "Specify this option to skip such tables") - .DefaultValue(defaults.SkipDocumentTables_).StoreResult(&SkipDocumentTables) - .Hidden(); // Deprecated - + + NDump::TRestoreSettings defaults; + + config.Opts->AddLongOption("restore-data", "Whether to restore data or not") + .DefaultValue(defaults.RestoreData_).StoreResult(&RestoreData); + + config.Opts->AddLongOption("restore-indexes", "Whether to restore indexes or not") + .DefaultValue(defaults.RestoreIndexes_).StoreResult(&RestoreIndexes); + + config.Opts->AddLongOption("skip-document-tables", TStringBuilder() + << "Document API tables cannot be restored for now. " + << "Specify this option to skip such tables") + .DefaultValue(defaults.SkipDocumentTables_).StoreResult(&SkipDocumentTables) + .Hidden(); // Deprecated + config.Opts->AddLongOption("save-partial-result", "Do not remove partial restore result." " If this option is not enabled, all changes in database that have already been applied during restore" " will be reverted in case of error.") .StoreTrue(&SavePartialResult); - config.Opts->AddLongOption("bandwidth", "Limit data upload bandwidth, bytes per second (example: 2MiB)") + config.Opts->AddLongOption("bandwidth", "Limit data upload bandwidth, bytes per second (example: 2MiB)") .DefaultValue("0").StoreResult(&UploadBandwidth); - - config.Opts->AddLongOption("rps", "Limit requests per second (example: 100)") - .DefaultValue(defaults.RateLimiterSettings_.GetRps()).StoreResult(&UploadRps); - - config.Opts->AddLongOption("upload-batch-rows", "Limit upload batch size in rows (example: 1K)") - .DefaultValue(defaults.RowsPerRequest_).StoreResult(&RowsPerRequest); - - config.Opts->AddLongOption("upload-batch-bytes", "Limit upload batch size in bytes (example: 1MiB)") - .DefaultValue(HumanReadableSize(defaults.BytesPerRequest_, SF_BYTES)).StoreResult(&BytesPerRequest); - - config.Opts->AddLongOption("upload-batch-rus", "Limit upload batch size in request units (example: 100)") - .DefaultValue(defaults.RequestUnitsPerRequest_).StoreResult(&RequestUnitsPerRequest); - + + config.Opts->AddLongOption("rps", "Limit requests per second (example: 100)") + .DefaultValue(defaults.RateLimiterSettings_.GetRps()).StoreResult(&UploadRps); + + config.Opts->AddLongOption("upload-batch-rows", "Limit upload batch size in rows (example: 1K)") + .DefaultValue(defaults.RowsPerRequest_).StoreResult(&RowsPerRequest); + + config.Opts->AddLongOption("upload-batch-bytes", "Limit upload batch size in bytes (example: 1MiB)") + .DefaultValue(HumanReadableSize(defaults.BytesPerRequest_, SF_BYTES)).StoreResult(&BytesPerRequest); + + config.Opts->AddLongOption("upload-batch-rus", "Limit upload batch size in request units (example: 100)") + .DefaultValue(defaults.RequestUnitsPerRequest_).StoreResult(&RequestUnitsPerRequest); + config.Opts->AddLongOption("in-flight", "Limit in-flight request count") - .DefaultValue(defaults.InFly_).StoreResult(&InFly); - + .DefaultValue(defaults.InFly_).StoreResult(&InFly); + config.Opts->AddLongOption("bulk-upsert", "Use BulkUpsert - a more efficient way to upload data with lower consistency level." " Global secondary indexes are not supported in this mode.") .StoreTrue(&UseBulkUpsert) .Hidden(); // Deprecated. Using ImportData should be more effective. - + config.Opts->AddLongOption("import-data", "Use ImportData - a more efficient way to upload data with lower consistency level." " Global secondary indexes are not supported in this mode.") - .StoreTrue(&UseImportData); - - config.Opts->MutuallyExclusive("bandwidth", "rps"); - config.Opts->MutuallyExclusive("import-data", "bulk-upsert"); + .StoreTrue(&UseImportData); + + config.Opts->MutuallyExclusive("bandwidth", "rps"); + config.Opts->MutuallyExclusive("import-data", "bulk-upsert"); } void TCommandRestore::Parse(TConfig& config) { @@ -186,44 +186,44 @@ void TCommandRestore::Parse(TConfig& config) { int TCommandRestore::Run(TConfig& config) { NYdb::SetVerbosity(config.IsVerbose); - auto settings = NDump::TRestoreSettings() - .DryRun(IsDryRun) - .RestoreData(RestoreData) - .RestoreIndexes(RestoreIndexes) - .SkipDocumentTables(SkipDocumentTables) + auto settings = NDump::TRestoreSettings() + .DryRun(IsDryRun) + .RestoreData(RestoreData) + .RestoreIndexes(RestoreIndexes) + .SkipDocumentTables(SkipDocumentTables) .SavePartialResult(SavePartialResult) - .RowsPerRequest(NYdb::SizeFromString(RowsPerRequest)) - .InFly(InFly); - - if (auto bytesPerRequest = NYdb::SizeFromString(BytesPerRequest)) { - if (bytesPerRequest > NDump::TRestoreSettings::MaxBytesPerRequest) { - throw TMissUseException() - << "--upload-batch-bytes cannot be larger than " - << HumanReadableSize(NDump::TRestoreSettings::MaxBytesPerRequest, SF_BYTES); - } - - settings.BytesPerRequest(bytesPerRequest); + .RowsPerRequest(NYdb::SizeFromString(RowsPerRequest)) + .InFly(InFly); + + if (auto bytesPerRequest = NYdb::SizeFromString(BytesPerRequest)) { + if (bytesPerRequest > NDump::TRestoreSettings::MaxBytesPerRequest) { + throw TMissUseException() + << "--upload-batch-bytes cannot be larger than " + << HumanReadableSize(NDump::TRestoreSettings::MaxBytesPerRequest, SF_BYTES); + } + + settings.BytesPerRequest(bytesPerRequest); + } + + if (RequestUnitsPerRequest) { + settings.RequestUnitsPerRequest(NYdb::SizeFromString(RequestUnitsPerRequest)); } - - if (RequestUnitsPerRequest) { - settings.RequestUnitsPerRequest(NYdb::SizeFromString(RequestUnitsPerRequest)); - } - - if (auto bandwidth = NYdb::SizeFromString(UploadBandwidth)) { - settings.RateLimiterSettings_.WithBandwidth(bandwidth, settings.BytesPerRequest_); - } else if (auto rps = NYdb::SizeFromString(UploadRps)) { - settings.RateLimiterSettings_.WithRps(rps); - } - - if (UseBulkUpsert) { - settings.Mode(NDump::TRestoreSettings::EMode::BulkUpsert); - } else if (UseImportData) { - settings.Mode(NDump::TRestoreSettings::EMode::ImportData); - } - - NDump::TClient client(CreateDriver(config)); - ThrowOnError(client.Restore(FilePath, Path, settings)); - + + if (auto bandwidth = NYdb::SizeFromString(UploadBandwidth)) { + settings.RateLimiterSettings_.WithBandwidth(bandwidth, settings.BytesPerRequest_); + } else if (auto rps = NYdb::SizeFromString(UploadRps)) { + settings.RateLimiterSettings_.WithRps(rps); + } + + if (UseBulkUpsert) { + settings.Mode(NDump::TRestoreSettings::EMode::BulkUpsert); + } else if (UseImportData) { + settings.Mode(NDump::TRestoreSettings::EMode::ImportData); + } + + NDump::TClient client(CreateDriver(config)); + ThrowOnError(client.Restore(FilePath, Path, settings)); + return EXIT_SUCCESS; } @@ -233,12 +233,12 @@ int TCommandRestore::Run(TConfig& config) { TCommandCopy::TCommandCopy() : TTableCommand("copy", {}, "Copy table(s)") -{ - TItem::DefineFields({ - {"Source", {{"source", "src", "s"}, "Source table path", true}}, - {"Destination", {{"destination", "dst", "d"}, "Destination table path", true}} - }); -} +{ + TItem::DefineFields({ + {"Source", {{"source", "src", "s"}, "Source table path", true}}, + {"Destination", {{"destination", "dst", "d"}, "Destination table path", true}} + }); +} void TCommandCopy::Config(TConfig& config) { TTableCommand::Config(config); @@ -248,7 +248,7 @@ void TCommandCopy::Config(TConfig& config) { TStringBuilder itemHelp; itemHelp << "[At least one] Item specification" << Endl << " Possible property names:" << Endl - << TItem::FormatHelp(2); + << TItem::FormatHelp(2); config.Opts->AddLongOption("item", itemHelp) .RequiredArgument("PROPERTY=VALUE,..."); } @@ -260,11 +260,11 @@ void TCommandCopy::Parse(TConfig& config) { if (Items.empty()) { throw TMissUseException() << "At least one item should be provided"; } - - for (auto& item : Items) { - NConsoleClient::AdjustPath(item.Source, config); - NConsoleClient::AdjustPath(item.Destination, config); - } + + for (auto& item : Items) { + NConsoleClient::AdjustPath(item.Source, config); + NConsoleClient::AdjustPath(item.Destination, config); + } } int TCommandCopy::Run(TConfig& config) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.h b/ydb/public/lib/ydb_cli/commands/ydb_tools.h index 48169816993..3a1218a9816 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.h @@ -7,8 +7,8 @@ #include <ydb/public/lib/ydb_cli/common/examples.h> #include <ydb/public/lib/ydb_cli/common/parseable_struct.h> -#include <library/cpp/regex/pcre/regexp.h> - +#include <library/cpp/regex/pcre/regexp.h> + namespace NYdb { namespace NConsoleClient { @@ -36,13 +36,13 @@ public: virtual int Run(TConfig& config) override; private: - TVector<TRegExMatch> ExclusionPatterns; + TVector<TRegExMatch> ExclusionPatterns; TString FilePath; bool IsSchemeOnly; - bool AvoidCopy = false; + bool AvoidCopy = false; bool SavePartialResult = false; TString ConsistencyLevel; - bool PreservePoolKinds = false; + bool PreservePoolKinds = false; }; class TCommandRestore : public TToolsCommand, public TCommandWithPath { @@ -55,18 +55,18 @@ public: private: TString FilePath; bool IsDryRun = false; - bool RestoreData = true; - bool RestoreIndexes = true; - bool SkipDocumentTables = false; + bool RestoreData = true; + bool RestoreIndexes = true; + bool SkipDocumentTables = false; bool SavePartialResult = false; TString UploadBandwidth; - TString UploadRps; - TString RowsPerRequest; - TString BytesPerRequest; - TString RequestUnitsPerRequest; - ui32 InFly; + TString UploadRps; + TString RowsPerRequest; + TString BytesPerRequest; + TString RequestUnitsPerRequest; + ui32 InFly; bool UseBulkUpsert = false; - bool UseImportData = false; + bool UseImportData = false; }; class TCommandCopy : public TTableCommand { diff --git a/ydb/public/lib/ydb_cli/common/aws.cpp b/ydb/public/lib/ydb_cli/common/aws.cpp index 9f98462adaf..8e73b772b91 100644 --- a/ydb/public/lib/ydb_cli/common/aws.cpp +++ b/ydb/public/lib/ydb_cli/common/aws.cpp @@ -1,53 +1,53 @@ -#include "aws.h" - -namespace NYdb { -namespace NConsoleClient { - -const TString TCommandWithAwsCredentials::AwsCredentialsFile= "~/.aws/credentials"; - -const TString& TCommandWithAwsCredentials::ReadAwsCredentialsFile() { - if (!FileContent) { - TString credentialsFile = AwsCredentialsFile; - FileContent = ReadFromFile(credentialsFile, "AWS Credentials"); - } - - return FileContent.GetRef(); -} - -static bool IsSkipSymbol(char s) { - switch (s) { - case ' ': - case '\t': - case '=': - return true; - } - return false; -} - -static TStringBuf GetKey(TStringBuf content, const TString& key) { - TStringBuf line; - while (content.ReadLine(line)) { - if (!line.SkipPrefix(key)) { - continue; - } - - while (!line.empty() && IsSkipSymbol(line.front())) { - line.Skip(1); - } - - return line; - } - - throw TMissUseException() << "Cannot find \"" << key << "\" key"; -} - -void TCommandWithAwsCredentials::ReadAwsAccessKey() { - AwsAccessKey = GetKey(ReadAwsCredentialsFile(), "aws_access_key_id"); -} - -void TCommandWithAwsCredentials::ReadAwsSecretKey() { - AwsSecretKey = GetKey(ReadAwsCredentialsFile(), "aws_secret_access_key"); -} - -} -} +#include "aws.h" + +namespace NYdb { +namespace NConsoleClient { + +const TString TCommandWithAwsCredentials::AwsCredentialsFile= "~/.aws/credentials"; + +const TString& TCommandWithAwsCredentials::ReadAwsCredentialsFile() { + if (!FileContent) { + TString credentialsFile = AwsCredentialsFile; + FileContent = ReadFromFile(credentialsFile, "AWS Credentials"); + } + + return FileContent.GetRef(); +} + +static bool IsSkipSymbol(char s) { + switch (s) { + case ' ': + case '\t': + case '=': + return true; + } + return false; +} + +static TStringBuf GetKey(TStringBuf content, const TString& key) { + TStringBuf line; + while (content.ReadLine(line)) { + if (!line.SkipPrefix(key)) { + continue; + } + + while (!line.empty() && IsSkipSymbol(line.front())) { + line.Skip(1); + } + + return line; + } + + throw TMissUseException() << "Cannot find \"" << key << "\" key"; +} + +void TCommandWithAwsCredentials::ReadAwsAccessKey() { + AwsAccessKey = GetKey(ReadAwsCredentialsFile(), "aws_access_key_id"); +} + +void TCommandWithAwsCredentials::ReadAwsSecretKey() { + AwsSecretKey = GetKey(ReadAwsCredentialsFile(), "aws_secret_access_key"); +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/aws.h b/ydb/public/lib/ydb_cli/common/aws.h index 104454d7855..5d793b6b384 100644 --- a/ydb/public/lib/ydb_cli/common/aws.h +++ b/ydb/public/lib/ydb_cli/common/aws.h @@ -1,62 +1,62 @@ -#pragma once - -#include "command.h" - -#include <util/generic/maybe.h> -#include <util/system/env.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandWithAwsCredentials { - template <typename TOpt, typename TReaderFunc> - static void Parse( - const TClientCommand::TConfig& config, - const TOpt opt, - const TString& envKey, - TString& parsed, - TReaderFunc readerFunc) { - - if (config.ParseResult->Has(opt)) { - parsed = config.ParseResult->Get(opt); - } else { - TString fromEnv = GetEnv(envKey); - - if (!fromEnv.empty()) { - parsed = std::move(fromEnv); - } else { - readerFunc(); - } - } - } - -protected: - template <typename TOpt> - void ParseAwsAccessKey(const TClientCommand::TConfig& config, const TOpt opt) { - Parse(config, opt, "AWS_ACCESS_KEY_ID", AwsAccessKey, [this]() { - ReadAwsAccessKey(); - }); - } - - template <typename TOpt> - void ParseAwsSecretKey(const TClientCommand::TConfig& config, const TOpt opt) { - Parse(config, opt, "AWS_SECRET_ACCESS_KEY", AwsSecretKey, [this]() { - ReadAwsSecretKey(); - }); - } - - TString AwsAccessKey; - TString AwsSecretKey; - - static const TString AwsCredentialsFile; - -private: - const TString& ReadAwsCredentialsFile(); - void ReadAwsAccessKey(); - void ReadAwsSecretKey(); - - TMaybe<TString> FileContent; -}; - -} -} +#pragma once + +#include "command.h" + +#include <util/generic/maybe.h> +#include <util/system/env.h> + +namespace NYdb { +namespace NConsoleClient { + +class TCommandWithAwsCredentials { + template <typename TOpt, typename TReaderFunc> + static void Parse( + const TClientCommand::TConfig& config, + const TOpt opt, + const TString& envKey, + TString& parsed, + TReaderFunc readerFunc) { + + if (config.ParseResult->Has(opt)) { + parsed = config.ParseResult->Get(opt); + } else { + TString fromEnv = GetEnv(envKey); + + if (!fromEnv.empty()) { + parsed = std::move(fromEnv); + } else { + readerFunc(); + } + } + } + +protected: + template <typename TOpt> + void ParseAwsAccessKey(const TClientCommand::TConfig& config, const TOpt opt) { + Parse(config, opt, "AWS_ACCESS_KEY_ID", AwsAccessKey, [this]() { + ReadAwsAccessKey(); + }); + } + + template <typename TOpt> + void ParseAwsSecretKey(const TClientCommand::TConfig& config, const TOpt opt) { + Parse(config, opt, "AWS_SECRET_ACCESS_KEY", AwsSecretKey, [this]() { + ReadAwsSecretKey(); + }); + } + + TString AwsAccessKey; + TString AwsSecretKey; + + static const TString AwsCredentialsFile; + +private: + const TString& ReadAwsCredentialsFile(); + void ReadAwsAccessKey(); + void ReadAwsSecretKey(); + + TMaybe<TString> FileContent; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 2e377b58c11..ce7334d7388 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -95,7 +95,7 @@ TClientCommand::TClientCommand(const TString& name, const std::initializer_list< HideOption("svnrevision"); Opts.AddHelpOption('h'); ChangeOptionDescription("help", "Print usage"); - Opts.SetWrap(Max(Opts.Wrap_, static_cast<ui32>(TermWidth()))); + Opts.SetWrap(Max(Opts.Wrap_, static_cast<ui32>(TermWidth()))); } @@ -370,7 +370,7 @@ void TCommandWithPath::AdjustPath(const TClientCommand::TConfig& config) { throw TMissUseException() << "Missing required argument <path>"; } - NConsoleClient::AdjustPath(Path, config); + NConsoleClient::AdjustPath(Path, config); } void TCommandWithStreamName::ParseStreamName(const TClientCommand::TConfig &config, const size_t argPos) { diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 4ad0bd0d521..7184ab41d1f 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -1,14 +1,14 @@ -#include "format.h" +#include "format.h" #include "pretty_table.h" - + #include <util/string/vector.h> #include <library/cpp/json/json_prettifier.h> #include <ydb/public/lib/json_value/ydb_json_value.h> -namespace NYdb { -namespace NConsoleClient { - +namespace NYdb { +namespace NConsoleClient { + namespace { THashMap<EOutputFormat, TString> InputFormatDescriptions = { { EOutputFormat::JsonUnicode, "Input in json format, binary strings are decoded with unicode characters" }, @@ -87,13 +87,13 @@ void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, const TVect auto findResult = FormatDescriptions.find(format); Y_VERIFY(findResult != FormatDescriptions.end(), "Couldn't find description for %s output format", (TStringBuilder() << format).c_str()); - description << "\n " << colors.BoldColor() << format << colors.OldColor() - << "\n " << findResult->second; + description << "\n " << colors.BoldColor() << format << colors.OldColor() + << "\n " << findResult->second; } config.Opts->AddLongOption("format", description.Str()) .RequiredArgument("STRING").StoreResult(&OutputFormat); AllowedFormats = allowedFormats; -} +} void TCommandWithFormat::ParseFormats() { if (InputFormat != EOutputFormat::Default @@ -108,7 +108,7 @@ void TCommandWithFormat::ParseFormats() { if (std::find(AllowedFormats.begin(), AllowedFormats.end(), OutputFormat) == AllowedFormats.end()) { throw TMissUseException() << "Output format " << OutputFormat << " is not available for this command"; } -} +} void TQueryPlanPrinter::Print(const TString& plan) { switch (Format) { diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index ae506255f4e..2d8d6a2e38b 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -1,15 +1,15 @@ -#pragma once - -#include "command.h" -#include "formats.h" - +#pragma once + +#include "command.h" +#include "formats.h" + #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> -namespace NYdb { -namespace NConsoleClient { - +namespace NYdb { +namespace NConsoleClient { + class TCommandWithResponseHeaders { protected: void PrintResponseHeader(const NYdb::TStatus& status); @@ -19,8 +19,8 @@ protected: static const TString ResponseHeadersHelp; }; -class TCommandWithFormat { -protected: +class TCommandWithFormat { +protected: void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); void ParseFormats(); @@ -29,8 +29,8 @@ protected: void AddJsonOption(TClientCommand::TConfig& config, const TString& description = "(Deprecated, will be removed soon. Use --format option instead)" " Output in json format"); - -protected: + +protected: EOutputFormat OutputFormat = EOutputFormat::Default; EOutputFormat InputFormat = EOutputFormat::Default; @@ -38,8 +38,8 @@ private: TVector<EOutputFormat> AllowedInputFormats; TVector<EOutputFormat> AllowedFormats; bool DeprecatedOptionUsed = false; -}; - +}; + class TResultSetPrinter { public: TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted = []() { return false; }); @@ -83,5 +83,5 @@ private: bool AnalyzeMode; }; -} -} +} +} diff --git a/ydb/public/lib/ydb_cli/common/formats.h b/ydb/public/lib/ydb_cli/common/formats.h index 09b4911bd29..46e83d943d9 100644 --- a/ydb/public/lib/ydb_cli/common/formats.h +++ b/ydb/public/lib/ydb_cli/common/formats.h @@ -1,9 +1,9 @@ -#pragma once - -namespace NYdb { -namespace NConsoleClient { - -enum class EOutputFormat { +#pragma once + +namespace NYdb { +namespace NConsoleClient { + +enum class EOutputFormat { Default /* "default" */, Pretty /* "pretty" */, Json /* "json" (deprecated) */, @@ -14,7 +14,7 @@ enum class EOutputFormat { ProtoJsonBase64 /* "proto-json-base64" */, Csv /* "csv" */, Tsv /* "tsv" */, -}; - -} -} +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/normalize_path.cpp b/ydb/public/lib/ydb_cli/common/normalize_path.cpp index 6efea8b8d1d..7ceec940453 100644 --- a/ydb/public/lib/ydb_cli/common/normalize_path.cpp +++ b/ydb/public/lib/ydb_cli/common/normalize_path.cpp @@ -44,23 +44,23 @@ namespace NConsoleClient { return result; } - void AdjustPath(TString& path, const TClientCommand::TConfig& config) { - if (path.StartsWith('/')) { - if (!path.StartsWith(config.Database)) { - throw TMissUseException() << "Provided path \"" << path << "\" starts with '/'. " - << "That means you are using an absolute path that should start with the path " - << "to your database \"" << config.Database << "\", but it doesn't. " << Endl - << "Please, provide full path starting from the domain root " - << "(example: \"/domain/my_base/dir1/table1\"). " << Endl - << "Or consider using relative path from your database (example: \"dir1/table1\")."; - } - } else { - // allow relative path - path = (config.Path ? config.Path : config.Database) + '/' + path; - } - - path = NormalizePath(path); - } - + void AdjustPath(TString& path, const TClientCommand::TConfig& config) { + if (path.StartsWith('/')) { + if (!path.StartsWith(config.Database)) { + throw TMissUseException() << "Provided path \"" << path << "\" starts with '/'. " + << "That means you are using an absolute path that should start with the path " + << "to your database \"" << config.Database << "\", but it doesn't. " << Endl + << "Please, provide full path starting from the domain root " + << "(example: \"/domain/my_base/dir1/table1\"). " << Endl + << "Or consider using relative path from your database (example: \"dir1/table1\")."; + } + } else { + // allow relative path + path = (config.Path ? config.Path : config.Database) + '/' + path; + } + + path = NormalizePath(path); + } + } } diff --git a/ydb/public/lib/ydb_cli/common/normalize_path.h b/ydb/public/lib/ydb_cli/common/normalize_path.h index e949cfd5e80..4352c907bca 100644 --- a/ydb/public/lib/ydb_cli/common/normalize_path.h +++ b/ydb/public/lib/ydb_cli/common/normalize_path.h @@ -1,10 +1,10 @@ #pragma once -#include "command.h" +#include "command.h" namespace NYdb { namespace NConsoleClient { TString NormalizePath(const TString &path); - void AdjustPath(TString& path, const TClientCommand::TConfig& config); + void AdjustPath(TString& path, const TClientCommand::TConfig& config); } } diff --git a/ydb/public/lib/ydb_cli/common/normalize_path_ut.cpp b/ydb/public/lib/ydb_cli/common/normalize_path_ut.cpp index 2f5a43dbb20..aae7cd7301e 100644 --- a/ydb/public/lib/ydb_cli/common/normalize_path_ut.cpp +++ b/ydb/public/lib/ydb_cli/common/normalize_path_ut.cpp @@ -4,7 +4,7 @@ using namespace NYdb::NConsoleClient; Y_UNIT_TEST_SUITE(NormalizePathTest) { - Y_UNIT_TEST(TestNormalization) { + Y_UNIT_TEST(TestNormalization) { UNIT_ASSERT(NormalizePath("/abc/./d") == "/abc/d"); UNIT_ASSERT(NormalizePath("/abc/./def//abcd//tre///brr") == "/abc/def/abcd/tre/brr"); UNIT_ASSERT(NormalizePath("/abc/../def") == "/abc/../def"); @@ -12,26 +12,26 @@ Y_UNIT_TEST_SUITE(NormalizePathTest) { UNIT_ASSERT(NormalizePath("/abc/def/.") == "/abc/def"); UNIT_ASSERT(NormalizePath("/abc/def/./") == "/abc/def"); } - - TString AdjustPath(const TString& path, const TClientCommand::TConfig& config) { - TString copyPath(path); - NYdb::NConsoleClient::AdjustPath(copyPath, config); - return copyPath; - } - - TClientCommand::TConfig FakeConfig(const TString& database) { - TClientCommand::TConfig config(0, nullptr); - config.Database = database; - return config; - } - - Y_UNIT_TEST(TestAdjustment) { - UNIT_ASSERT(AdjustPath("abc", FakeConfig("/root/db")) == "/root/db/abc"); - UNIT_ASSERT(AdjustPath("./abc", FakeConfig("/root/db")) == "/root/db/abc"); - UNIT_ASSERT(AdjustPath("/root/db/abc", FakeConfig("/root/db")) == "/root/db/abc"); - - UNIT_ASSERT_EXCEPTION(AdjustPath("/abc", FakeConfig("/root/db")), TMissUseException); - UNIT_ASSERT_EXCEPTION(AdjustPath("/root/bd/abc", FakeConfig("/root/db")), TMissUseException); - } + + TString AdjustPath(const TString& path, const TClientCommand::TConfig& config) { + TString copyPath(path); + NYdb::NConsoleClient::AdjustPath(copyPath, config); + return copyPath; + } + + TClientCommand::TConfig FakeConfig(const TString& database) { + TClientCommand::TConfig config(0, nullptr); + config.Database = database; + return config; + } + + Y_UNIT_TEST(TestAdjustment) { + UNIT_ASSERT(AdjustPath("abc", FakeConfig("/root/db")) == "/root/db/abc"); + UNIT_ASSERT(AdjustPath("./abc", FakeConfig("/root/db")) == "/root/db/abc"); + UNIT_ASSERT(AdjustPath("/root/db/abc", FakeConfig("/root/db")) == "/root/db/abc"); + + UNIT_ASSERT_EXCEPTION(AdjustPath("/abc", FakeConfig("/root/db")), TMissUseException); + UNIT_ASSERT_EXCEPTION(AdjustPath("/root/bd/abc", FakeConfig("/root/db")), TMissUseException); + } } diff --git a/ydb/public/lib/ydb_cli/common/parseable_struct.h b/ydb/public/lib/ydb_cli/common/parseable_struct.h index 4f2a502437d..6eeb08ffba9 100644 --- a/ydb/public/lib/ydb_cli/common/parseable_struct.h +++ b/ydb/public/lib/ydb_cli/common/parseable_struct.h @@ -1,196 +1,196 @@ -#pragma once - -#include "command.h" - +#pragma once + +#include "command.h" + #include <library/cpp/colorizer/colors.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/va_args.h> -#include <util/generic/vector.h> -#include <util/string/builder.h> -#include <util/string/cast.h> - -namespace NYdb { -namespace NConsoleClient { - -template <typename T> -class TParseableStruct { - using TOpt = NLastGetopt::TOpt; - using TOptParseResult = NLastGetopt::TOptParseResult; - using TOptsParseResult = NLastGetopt::TOptsParseResult; - - static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, const TOpt* opt) { - return parseResult->FindOptParseResult(opt); - } - - static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, const TString& name) { - return parseResult->FindLongOptParseResult(name); - } - - static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, char c) { - return parseResult->FindCharOptParseResult(c); - } - - static T FromString(const char* data) { - T t; - - TStringBuf buf(data); - TStringBuf property = buf.NextTok('='); - THashSet<TString> matched; - - while (property.IsInited()) { - if (const TString field = t.LoadProperty(property, buf.NextTok(','))) { - if (!matched.insert(field).second) { - throw TMissUseException() << "Duplicate value for \"" << field << "\""; - } - } else { - throw TMissUseException() << "Bad property: \"" << property << "\""; - } - - property = buf.NextTok('='); - } - - for (auto it = Fields.begin(), last = Fields.end(); it != last; ++it) { - if (it->second.Required && !matched.contains(it->first)) { - throw TMissUseException() << "Missing required property \"" << it->first << "\""; - } - } - - return t; - } - - static TString MakeIndent(size_t indentSize, char indentC = ' ') { - TStringBuilder indent; - - while (indentSize) { - indent << indentC; - --indentSize; - } - - return indent; - } - -public: - template <typename TOpt> - static TVector<T> Parse(const TClientCommand::TConfig& config, const TOpt opt) { - const auto* parseResult = FindOptParseResult(config.ParseResult, opt); - if (!parseResult) { - return {}; - } - - TVector<T> result(Reserve(parseResult->Count())); - - for (const char* value : parseResult->Values()) { - result.push_back(FromString(value)); - } - - return result; - } - - static TString FormatHelp(size_t indentSize = 0, char indentC = ' ') { - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - - TStringBuilder help; - const TString& indent = MakeIndent(indentSize, indentC); - - for (const auto& kv : Fields) { - help << indent << " " << colors.BoldColor() << kv.first << colors.OldColor(); - - if (kv.second.Aliases) { - help << " (aliases: "; - - bool first = true; - for (const auto& alias : kv.second.Aliases) { - if (!first) { - help << ", "; - } - first = false; - help << colors.BoldColor() << alias << colors.OldColor(); - } - - help << ")"; - } - - help << Endl; - - if (kv.second.Description) { - help << indent << " "; - if (kv.second.Required) { - help << "[Required] "; - } - help << kv.second.Description << Endl; - } - } - - return help; - } - -protected: - struct TField { - TVector<TString> Aliases; - TString Description; - bool Required; - }; - - static THashMap<TString, TField> Fields; - - static void DefineField(const TString& name, TField field) { - auto it = Fields.find(name); - Y_VERIFY(it != Fields.end()); - it->second = std::move(field); - } - - static bool MatchField(const TString& name, const TStringBuf property) { - if (name == property) { - return true; - } - - auto it = Fields.find(name); - Y_VERIFY(it != Fields.end()); - return Find(it->second.Aliases, property) != it->second.Aliases.end(); - } - -}; - -template <typename T> -THashMap<TString, typename TParseableStruct<T>::TField> TParseableStruct<T>::Fields; - -#if defined DEFINE_PARSEABLE_STRUCT -#error DEFINE_PARSEABLE_STRUCT macro redefinition -#endif - -#define DEFINE_PARSEABLE_STRUCT(name, fields, ...) \ - struct name : public fields, public TParseableStruct<name> { \ - static void DefineFields(THashMap<TString, TField> aliases) {\ - Y_PASS_VA_ARGS(Y_MAP_ARGS(__INIT_STRUCT_FIELD__, __VA_ARGS__)) \ - for (auto& kv : aliases) { \ - DefineField(kv.first, std::move(kv.second)); \ - } \ - } \ - TString LoadProperty(const TStringBuf property, const TStringBuf value) { \ - Y_PASS_VA_ARGS(Y_MAP_ARGS(__PARSE_STRUCT_FIELD__, __VA_ARGS__)) \ - return TString(); \ - } \ - } - -#if defined __INIT_STRUCT_FIELD__ -#error __INIT_STRUCT_FIELD__ macro redefinition -#endif - -#define __INIT_STRUCT_FIELD__(field) \ - Fields[#field] = {}; - -#if defined __PARSE_STRUCT_FIELD__ -#error __PARSE_STRUCT_FIELD__ macro redefinition -#endif - -#define __PARSE_STRUCT_FIELD__(field) \ - if (MatchField(#field, property)) { \ - field = ::FromString<decltype(field)>(value); \ - return #field; \ - } - -} -} + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/va_args.h> +#include <util/generic/vector.h> +#include <util/string/builder.h> +#include <util/string/cast.h> + +namespace NYdb { +namespace NConsoleClient { + +template <typename T> +class TParseableStruct { + using TOpt = NLastGetopt::TOpt; + using TOptParseResult = NLastGetopt::TOptParseResult; + using TOptsParseResult = NLastGetopt::TOptsParseResult; + + static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, const TOpt* opt) { + return parseResult->FindOptParseResult(opt); + } + + static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, const TString& name) { + return parseResult->FindLongOptParseResult(name); + } + + static const TOptParseResult* FindOptParseResult(const TOptsParseResult* parseResult, char c) { + return parseResult->FindCharOptParseResult(c); + } + + static T FromString(const char* data) { + T t; + + TStringBuf buf(data); + TStringBuf property = buf.NextTok('='); + THashSet<TString> matched; + + while (property.IsInited()) { + if (const TString field = t.LoadProperty(property, buf.NextTok(','))) { + if (!matched.insert(field).second) { + throw TMissUseException() << "Duplicate value for \"" << field << "\""; + } + } else { + throw TMissUseException() << "Bad property: \"" << property << "\""; + } + + property = buf.NextTok('='); + } + + for (auto it = Fields.begin(), last = Fields.end(); it != last; ++it) { + if (it->second.Required && !matched.contains(it->first)) { + throw TMissUseException() << "Missing required property \"" << it->first << "\""; + } + } + + return t; + } + + static TString MakeIndent(size_t indentSize, char indentC = ' ') { + TStringBuilder indent; + + while (indentSize) { + indent << indentC; + --indentSize; + } + + return indent; + } + +public: + template <typename TOpt> + static TVector<T> Parse(const TClientCommand::TConfig& config, const TOpt opt) { + const auto* parseResult = FindOptParseResult(config.ParseResult, opt); + if (!parseResult) { + return {}; + } + + TVector<T> result(Reserve(parseResult->Count())); + + for (const char* value : parseResult->Values()) { + result.push_back(FromString(value)); + } + + return result; + } + + static TString FormatHelp(size_t indentSize = 0, char indentC = ' ') { + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + + TStringBuilder help; + const TString& indent = MakeIndent(indentSize, indentC); + + for (const auto& kv : Fields) { + help << indent << " " << colors.BoldColor() << kv.first << colors.OldColor(); + + if (kv.second.Aliases) { + help << " (aliases: "; + + bool first = true; + for (const auto& alias : kv.second.Aliases) { + if (!first) { + help << ", "; + } + first = false; + help << colors.BoldColor() << alias << colors.OldColor(); + } + + help << ")"; + } + + help << Endl; + + if (kv.second.Description) { + help << indent << " "; + if (kv.second.Required) { + help << "[Required] "; + } + help << kv.second.Description << Endl; + } + } + + return help; + } + +protected: + struct TField { + TVector<TString> Aliases; + TString Description; + bool Required; + }; + + static THashMap<TString, TField> Fields; + + static void DefineField(const TString& name, TField field) { + auto it = Fields.find(name); + Y_VERIFY(it != Fields.end()); + it->second = std::move(field); + } + + static bool MatchField(const TString& name, const TStringBuf property) { + if (name == property) { + return true; + } + + auto it = Fields.find(name); + Y_VERIFY(it != Fields.end()); + return Find(it->second.Aliases, property) != it->second.Aliases.end(); + } + +}; + +template <typename T> +THashMap<TString, typename TParseableStruct<T>::TField> TParseableStruct<T>::Fields; + +#if defined DEFINE_PARSEABLE_STRUCT +#error DEFINE_PARSEABLE_STRUCT macro redefinition +#endif + +#define DEFINE_PARSEABLE_STRUCT(name, fields, ...) \ + struct name : public fields, public TParseableStruct<name> { \ + static void DefineFields(THashMap<TString, TField> aliases) {\ + Y_PASS_VA_ARGS(Y_MAP_ARGS(__INIT_STRUCT_FIELD__, __VA_ARGS__)) \ + for (auto& kv : aliases) { \ + DefineField(kv.first, std::move(kv.second)); \ + } \ + } \ + TString LoadProperty(const TStringBuf property, const TStringBuf value) { \ + Y_PASS_VA_ARGS(Y_MAP_ARGS(__PARSE_STRUCT_FIELD__, __VA_ARGS__)) \ + return TString(); \ + } \ + } + +#if defined __INIT_STRUCT_FIELD__ +#error __INIT_STRUCT_FIELD__ macro redefinition +#endif + +#define __INIT_STRUCT_FIELD__(field) \ + Fields[#field] = {}; + +#if defined __PARSE_STRUCT_FIELD__ +#error __PARSE_STRUCT_FIELD__ macro redefinition +#endif + +#define __PARSE_STRUCT_FIELD__(field) \ + if (MatchField(#field, property)) { \ + field = ::FromString<decltype(field)>(value); \ + return #field; \ + } + +} +} diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.cpp b/ydb/public/lib/ydb_cli/common/pretty_table.cpp index 1e921cfadc5..d376454198b 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.cpp +++ b/ydb/public/lib/ydb_cli/common/pretty_table.cpp @@ -1,168 +1,168 @@ -#include "pretty_table.h" +#include "pretty_table.h" #include "common.h" - -#include <util/generic/algorithm.h> -#include <util/generic/xrange.h> -#include <util/stream/format.h> - -namespace NYdb { -namespace NConsoleClient { - -TPrettyTable::TRow::TRow(size_t nColumns) - : Columns(nColumns) -{ -} - -size_t TPrettyTable::TRow::ColumnWidth(size_t columnIndex) const { - Y_VERIFY(columnIndex < Columns.size()); - - size_t width = 0; - for (const auto& line : Columns.at(columnIndex)) { - width = Max(width, line.size()); - } - - return width; -} - -bool TPrettyTable::TRow::PrintColumns(IOutputStream& o, const TVector<size_t>& widths, size_t lineNumber) const { - bool next = false; - - for (size_t columnIndex : xrange(Columns.size())) { - if (columnIndex == 0) { + +#include <util/generic/algorithm.h> +#include <util/generic/xrange.h> +#include <util/stream/format.h> + +namespace NYdb { +namespace NConsoleClient { + +TPrettyTable::TRow::TRow(size_t nColumns) + : Columns(nColumns) +{ +} + +size_t TPrettyTable::TRow::ColumnWidth(size_t columnIndex) const { + Y_VERIFY(columnIndex < Columns.size()); + + size_t width = 0; + for (const auto& line : Columns.at(columnIndex)) { + width = Max(width, line.size()); + } + + return width; +} + +bool TPrettyTable::TRow::PrintColumns(IOutputStream& o, const TVector<size_t>& widths, size_t lineNumber) const { + bool next = false; + + for (size_t columnIndex : xrange(Columns.size())) { + if (columnIndex == 0) { o << "| "; - } else { + } else { o << " | "; - } - - if (const size_t width = widths.at(columnIndex)) { - const auto& column = Columns.at(columnIndex); - - TStringBuf data; - size_t l = 0; - for (const auto& line : column) { - data = line; - while (data && l < lineNumber) { - data.Skip(width); - ++l; - } - } - - if (data) { - o << RightPad(data.SubStr(0, width), width); - } else { - o << RightPad(' ', width); - } - - if (data.size() > width) { - next = true; - } - } - } + } + + if (const size_t width = widths.at(columnIndex)) { + const auto& column = Columns.at(columnIndex); + + TStringBuf data; + size_t l = 0; + for (const auto& line : column) { + data = line; + while (data && l < lineNumber) { + data.Skip(width); + ++l; + } + } + + if (data) { + o << RightPad(data.SubStr(0, width), width); + } else { + o << RightPad(' ', width); + } + + if (data.size() > width) { + next = true; + } + } + } o << " |" << Endl; - - return next; -} - -bool TPrettyTable::TRow::HasFreeText() const { - return !!Text; -} - -void TPrettyTable::TRow::PrintFreeText(IOutputStream& o, size_t width) const { - Y_VERIFY(HasFreeText()); - - for (auto& line : StringSplitter(Text).Split('\n')) { - TStringBuf token = line.Token(); - - while (token) { + + return next; +} + +bool TPrettyTable::TRow::HasFreeText() const { + return !!Text; +} + +void TPrettyTable::TRow::PrintFreeText(IOutputStream& o, size_t width) const { + Y_VERIFY(HasFreeText()); + + for (auto& line : StringSplitter(Text).Split('\n')) { + TStringBuf token = line.Token(); + + while (token) { o << "| " << RightPad(token.SubStr(0, width), width) << " |" << Endl; - token.Skip(width); - } - } -} - -TPrettyTable::TRow& TPrettyTable::AddRow() { - Rows.push_back(TRow(Columns)); - return Rows.back(); -} - -static void PrintDelim(IOutputStream& o, const TVector<size_t>& widths, - const TString& b, const TString& s, const TString& e, bool dotted = false) { - o << b; - - for (auto i : xrange(widths.size())) { - const size_t width = widths.at(i); - - for (auto x : xrange(width + 2)) { - Y_UNUSED(x); - o << (dotted ? "╴" : "─"); - } - - if (i != widths.size() - 1) { - o << s; - } - } - - o << e << Endl; -} - -void TPrettyTable::Print(IOutputStream& o) const { - if (!Rows) { - return; - } - - TVector<size_t> widths = CalcWidths(); - - PrintDelim(o, widths, "┌", "┬", "┐"); - for (auto i : xrange(Rows.size())) { - const auto& row = Rows.at(i); - - size_t line = 0; - while (row.PrintColumns(o, widths, line)) { - ++line; - } - - if (row.HasFreeText()) { - PrintDelim(o, widths, "├", "┴", "┤", true); - row.PrintFreeText(o, Accumulate(widths, (Columns - 1) * 3)); - } - - if ((Config.DelimRows && i != Rows.size() - 1) || (Config.Header && i == 0)) { - PrintDelim(o, widths, "├", row.HasFreeText() ? "┬" : "┼", "┤"); - } - } - PrintDelim(o, widths, "└", Rows.back().HasFreeText() ? "─" : "┴", "┘"); -} - -TVector<size_t> TPrettyTable::CalcWidths() const { - TVector<size_t> widths(Columns); - - // max - for (const auto& row : Rows) { - for (size_t columnIndex : xrange(Columns)) { - widths[columnIndex] = Max(widths[columnIndex], row.ColumnWidth(columnIndex)); - } - } - - // adjust - const size_t maxWidth = Max(Config.Width, TermWidth()) - ((Columns * 3) + 1); - size_t totalWidth = Accumulate(widths, (size_t)0); - while (totalWidth > maxWidth) { - auto it = MaxElement(widths.begin(), widths.end()); - if (*it == 1) { - break; - } - - if ((totalWidth - maxWidth) < (*it / 2)) { - *it = *it - (totalWidth - maxWidth); - } else { - *it = Max(*it / 2, (size_t)1); - } - - totalWidth = Accumulate(widths, 0); - } - - return widths; -} - -} -} + token.Skip(width); + } + } +} + +TPrettyTable::TRow& TPrettyTable::AddRow() { + Rows.push_back(TRow(Columns)); + return Rows.back(); +} + +static void PrintDelim(IOutputStream& o, const TVector<size_t>& widths, + const TString& b, const TString& s, const TString& e, bool dotted = false) { + o << b; + + for (auto i : xrange(widths.size())) { + const size_t width = widths.at(i); + + for (auto x : xrange(width + 2)) { + Y_UNUSED(x); + o << (dotted ? "╴" : "─"); + } + + if (i != widths.size() - 1) { + o << s; + } + } + + o << e << Endl; +} + +void TPrettyTable::Print(IOutputStream& o) const { + if (!Rows) { + return; + } + + TVector<size_t> widths = CalcWidths(); + + PrintDelim(o, widths, "┌", "┬", "┐"); + for (auto i : xrange(Rows.size())) { + const auto& row = Rows.at(i); + + size_t line = 0; + while (row.PrintColumns(o, widths, line)) { + ++line; + } + + if (row.HasFreeText()) { + PrintDelim(o, widths, "├", "┴", "┤", true); + row.PrintFreeText(o, Accumulate(widths, (Columns - 1) * 3)); + } + + if ((Config.DelimRows && i != Rows.size() - 1) || (Config.Header && i == 0)) { + PrintDelim(o, widths, "├", row.HasFreeText() ? "┬" : "┼", "┤"); + } + } + PrintDelim(o, widths, "└", Rows.back().HasFreeText() ? "─" : "┴", "┘"); +} + +TVector<size_t> TPrettyTable::CalcWidths() const { + TVector<size_t> widths(Columns); + + // max + for (const auto& row : Rows) { + for (size_t columnIndex : xrange(Columns)) { + widths[columnIndex] = Max(widths[columnIndex], row.ColumnWidth(columnIndex)); + } + } + + // adjust + const size_t maxWidth = Max(Config.Width, TermWidth()) - ((Columns * 3) + 1); + size_t totalWidth = Accumulate(widths, (size_t)0); + while (totalWidth > maxWidth) { + auto it = MaxElement(widths.begin(), widths.end()); + if (*it == 1) { + break; + } + + if ((totalWidth - maxWidth) < (*it / 2)) { + *it = *it - (totalWidth - maxWidth); + } else { + *it = Max(*it / 2, (size_t)1); + } + + totalWidth = Accumulate(widths, 0); + } + + return widths; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.h b/ydb/public/lib/ydb_cli/common/pretty_table.h index 761ea9df26b..f39411d9020 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.h +++ b/ydb/public/lib/ydb_cli/common/pretty_table.h @@ -1,106 +1,106 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/string/builder.h> -#include <util/string/split.h> - -namespace NYdb { -namespace NConsoleClient { - -struct TPrettyTableConfig { - bool Header = true; - bool DelimRows = true; - size_t Width = 0; // auto - - TPrettyTableConfig& WithoutHeader() { - Header = false; - return *this; - } - - TPrettyTableConfig& WithoutRowDelimiters() { - DelimRows = false; - return *this; - } - - TPrettyTableConfig& MaxWidth(size_t width) { - Width = width; - return *this; - } -}; - -class TPrettyTable { -public: - class TRow { - friend class TPrettyTable; - +#pragma once + +#include <util/generic/vector.h> +#include <util/string/builder.h> +#include <util/string/split.h> + +namespace NYdb { +namespace NConsoleClient { + +struct TPrettyTableConfig { + bool Header = true; + bool DelimRows = true; + size_t Width = 0; // auto + + TPrettyTableConfig& WithoutHeader() { + Header = false; + return *this; + } + + TPrettyTableConfig& WithoutRowDelimiters() { + DelimRows = false; + return *this; + } + + TPrettyTableConfig& MaxWidth(size_t width) { + Width = width; + return *this; + } +}; + +class TPrettyTable { +public: + class TRow { + friend class TPrettyTable; + explicit TRow(const TVector<TString>& columnNames) { - for (const auto& name : columnNames) { - Columns.push_back({name}); - } - } - - explicit TRow(size_t nColumns); - - public: - template <typename T> - TRow& Column(size_t index, const T& data) { - TString lines = TStringBuilder() << data; - - for (auto& line : StringSplitter(lines).Split('\n')) { - if (line.Empty()) { - continue; - } - - Columns[index].emplace_back(std::move(line)); - } - - return *this; - } - - TRow& FreeText(TString text) { - Text = std::move(text); - return *this; - } - - private: - size_t ColumnWidth(size_t columnIndex) const; - bool PrintColumns(IOutputStream& o, const TVector<size_t>& widths, size_t lineNumber) const; - bool HasFreeText() const; - void PrintFreeText(IOutputStream& o, size_t width) const; - - private: - TVector<TVector<TString>> Columns; - TString Text; - - }; - -public: + for (const auto& name : columnNames) { + Columns.push_back({name}); + } + } + + explicit TRow(size_t nColumns); + + public: + template <typename T> + TRow& Column(size_t index, const T& data) { + TString lines = TStringBuilder() << data; + + for (auto& line : StringSplitter(lines).Split('\n')) { + if (line.Empty()) { + continue; + } + + Columns[index].emplace_back(std::move(line)); + } + + return *this; + } + + TRow& FreeText(TString text) { + Text = std::move(text); + return *this; + } + + private: + size_t ColumnWidth(size_t columnIndex) const; + bool PrintColumns(IOutputStream& o, const TVector<size_t>& widths, size_t lineNumber) const; + bool HasFreeText() const; + void PrintFreeText(IOutputStream& o, size_t width) const; + + private: + TVector<TVector<TString>> Columns; + TString Text; + + }; + +public: explicit TPrettyTable(const TVector<TString>& columnNames, const TPrettyTableConfig& config = TPrettyTableConfig()) - : Columns(columnNames.size()) - , Config(config) - { - if (Config.Header) { - Rows.push_back(TRow(columnNames)); - } - } - - TRow& AddRow(); - void Print(IOutputStream& o) const; - -private: - TVector<size_t> CalcWidths() const; - -private: - const size_t Columns; - const TPrettyTableConfig Config; - TVector<TRow> Rows; - -}; - -} -} - -template <> -inline void Out<NYdb::NConsoleClient::TPrettyTable>(IOutputStream& o, const NYdb::NConsoleClient::TPrettyTable& x) { - return x.Print(o); -} + : Columns(columnNames.size()) + , Config(config) + { + if (Config.Header) { + Rows.push_back(TRow(columnNames)); + } + } + + TRow& AddRow(); + void Print(IOutputStream& o) const; + +private: + TVector<size_t> CalcWidths() const; + +private: + const size_t Columns; + const TPrettyTableConfig Config; + TVector<TRow> Rows; + +}; + +} +} + +template <> +inline void Out<NYdb::NConsoleClient::TPrettyTable>(IOutputStream& o, const NYdb::NConsoleClient::TPrettyTable& x) { + return x.Print(o); +} diff --git a/ydb/public/lib/ydb_cli/common/print_operation.cpp b/ydb/public/lib/ydb_cli/common/print_operation.cpp index 147dce1b583..28a84375dcd 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.cpp +++ b/ydb/public/lib/ydb_cli/common/print_operation.cpp @@ -1,19 +1,19 @@ -#include "print_operation.h" -#include "pretty_table.h" - +#include "print_operation.h" +#include "pretty_table.h" + #include <ydb/public/lib/operation_id/operation_id.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> - -#include <util/string/builder.h> -#include <util/string/cast.h> - -namespace NYdb { -namespace NConsoleClient { - -namespace { - - using namespace NKikimr::NOperationId; - + +#include <util/string/builder.h> +#include <util/string/cast.h> + +namespace NYdb { +namespace NConsoleClient { + +namespace { + + using namespace NKikimr::NOperationId; + /// Common TPrettyTable MakeTable(const TOperation&) { return TPrettyTable({"id", "ready", "status"}); @@ -40,153 +40,153 @@ namespace { row.FreeText(freeText); } - template <typename EProgress, typename TMetadata> - TString PrintProgress(const TMetadata& metadata) { - TStringBuilder result; - - result << metadata.Progress; - if (metadata.Progress != EProgress::TransferData) { - return result; - } - - if (!metadata.ItemsProgress) { - return result; - } - - ui32 partsTotal = 0; - ui32 partsCompleted = 0; - for (const auto& item : metadata.ItemsProgress) { - if (!item.PartsTotal) { - return result; - } - - partsTotal += item.PartsTotal; - partsCompleted += item.PartsCompleted; - } - - float percentage = float(partsCompleted) / partsTotal * 100; - result << " (" << FloatToString(percentage, PREC_POINT_DIGITS, 2) + "%)"; - - return result; - } - - /// YT - TPrettyTable MakeTable(const NExport::TExportToYtResponse&) { - return TPrettyTable({"id", "ready", "status", "progress", "yt proxy"}); - } - - void PrettyPrint(const NExport::TExportToYtResponse& operation, TPrettyTable& table) { - const auto& status = operation.Status(); - const auto& metadata = operation.Metadata(); - const auto& settings = metadata.Settings; - - auto& row = table.AddRow(); - row - .Column(0, ProtoToString(operation.Id())) - .Column(1, operation.Ready() ? "true" : "false") - .Column(2, status.GetStatus()) - .Column(3, PrintProgress<decltype(metadata.Progress)>(metadata)) - .Column(4, TStringBuilder() << settings.Host_ << ":" << settings.Port_.GetOrElse(80)); - - TStringBuilder freeText; - - if (!status.GetIssues().Empty()) { - freeText << "Issues: " << Endl; - for (const auto& issue : status.GetIssues()) { - freeText << " - " << issue << Endl; - } - } - - freeText << "Items: " << Endl; - for (const auto& item : settings.Item_) { - freeText - << " - source: " << item.Src << Endl - << " destination: " << item.Dst << Endl; - } - - if (settings.Description_) { - freeText << "Description: " << settings.Description_.GetRef() << Endl; - } - - if (settings.NumberOfRetries_) { - freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; - } - - freeText << "TypeV3: " << (settings.UseTypeV3_ ? "true" : "false") << Endl; - - row.FreeText(freeText); - } - - /// S3 - TPrettyTable MakeTableS3() { - return TPrettyTable({"id", "ready", "status", "progress", "endpoint", "bucket"}); - } - - template <typename T> - void PrettyPrintS3(const T& operation, TPrettyTable& table) { - const auto& status = operation.Status(); - const auto& metadata = operation.Metadata(); - const auto& settings = metadata.Settings; - - auto& row = table.AddRow(); - row - .Column(0, ProtoToString(operation.Id())) - .Column(1, operation.Ready() ? "true" : "false") - .Column(2, status.GetStatus()) - .Column(3, PrintProgress<decltype(metadata.Progress)>(metadata)) - .Column(4, settings.Endpoint_) - .Column(5, settings.Bucket_); - - TStringBuilder freeText; - - if constexpr (std::is_same_v<NExport::TExportToS3Response, T>) { - freeText << "StorageClass: " << settings.StorageClass_ << Endl; - } - - if (!status.GetIssues().Empty()) { - freeText << "Issues: " << Endl; - for (const auto& issue : status.GetIssues()) { - freeText << " - " << issue << Endl; - } - } - - freeText << "Items: " << Endl; - for (const auto& item : settings.Item_) { - freeText - << " - source: " << item.Src << Endl - << " destination: " << item.Dst << Endl; - } - - if (settings.Description_) { - freeText << "Description: " << settings.Description_.GetRef() << Endl; - } - - if (settings.NumberOfRetries_) { - freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; - } - - row.FreeText(freeText); - } - - // export - TPrettyTable MakeTable(const NExport::TExportToS3Response&) { - return MakeTableS3(); - } - - void PrettyPrint(const NExport::TExportToS3Response& operation, TPrettyTable& table) { - PrettyPrintS3(operation, table); - } - - // import - TPrettyTable MakeTable(const NImport::TImportFromS3Response&) { - return MakeTableS3(); - } - - void PrettyPrint(const NImport::TImportFromS3Response& operation, TPrettyTable& table) { - PrettyPrintS3(operation, table); - } - - /// Index build + template <typename EProgress, typename TMetadata> + TString PrintProgress(const TMetadata& metadata) { + TStringBuilder result; + + result << metadata.Progress; + if (metadata.Progress != EProgress::TransferData) { + return result; + } + + if (!metadata.ItemsProgress) { + return result; + } + + ui32 partsTotal = 0; + ui32 partsCompleted = 0; + for (const auto& item : metadata.ItemsProgress) { + if (!item.PartsTotal) { + return result; + } + + partsTotal += item.PartsTotal; + partsCompleted += item.PartsCompleted; + } + + float percentage = float(partsCompleted) / partsTotal * 100; + result << " (" << FloatToString(percentage, PREC_POINT_DIGITS, 2) + "%)"; + + return result; + } + + /// YT + TPrettyTable MakeTable(const NExport::TExportToYtResponse&) { + return TPrettyTable({"id", "ready", "status", "progress", "yt proxy"}); + } + + void PrettyPrint(const NExport::TExportToYtResponse& operation, TPrettyTable& table) { + const auto& status = operation.Status(); + const auto& metadata = operation.Metadata(); + const auto& settings = metadata.Settings; + + auto& row = table.AddRow(); + row + .Column(0, ProtoToString(operation.Id())) + .Column(1, operation.Ready() ? "true" : "false") + .Column(2, status.GetStatus()) + .Column(3, PrintProgress<decltype(metadata.Progress)>(metadata)) + .Column(4, TStringBuilder() << settings.Host_ << ":" << settings.Port_.GetOrElse(80)); + + TStringBuilder freeText; + + if (!status.GetIssues().Empty()) { + freeText << "Issues: " << Endl; + for (const auto& issue : status.GetIssues()) { + freeText << " - " << issue << Endl; + } + } + + freeText << "Items: " << Endl; + for (const auto& item : settings.Item_) { + freeText + << " - source: " << item.Src << Endl + << " destination: " << item.Dst << Endl; + } + + if (settings.Description_) { + freeText << "Description: " << settings.Description_.GetRef() << Endl; + } + + if (settings.NumberOfRetries_) { + freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; + } + + freeText << "TypeV3: " << (settings.UseTypeV3_ ? "true" : "false") << Endl; + + row.FreeText(freeText); + } + + /// S3 + TPrettyTable MakeTableS3() { + return TPrettyTable({"id", "ready", "status", "progress", "endpoint", "bucket"}); + } + + template <typename T> + void PrettyPrintS3(const T& operation, TPrettyTable& table) { + const auto& status = operation.Status(); + const auto& metadata = operation.Metadata(); + const auto& settings = metadata.Settings; + + auto& row = table.AddRow(); + row + .Column(0, ProtoToString(operation.Id())) + .Column(1, operation.Ready() ? "true" : "false") + .Column(2, status.GetStatus()) + .Column(3, PrintProgress<decltype(metadata.Progress)>(metadata)) + .Column(4, settings.Endpoint_) + .Column(5, settings.Bucket_); + + TStringBuilder freeText; + + if constexpr (std::is_same_v<NExport::TExportToS3Response, T>) { + freeText << "StorageClass: " << settings.StorageClass_ << Endl; + } + + if (!status.GetIssues().Empty()) { + freeText << "Issues: " << Endl; + for (const auto& issue : status.GetIssues()) { + freeText << " - " << issue << Endl; + } + } + + freeText << "Items: " << Endl; + for (const auto& item : settings.Item_) { + freeText + << " - source: " << item.Src << Endl + << " destination: " << item.Dst << Endl; + } + + if (settings.Description_) { + freeText << "Description: " << settings.Description_.GetRef() << Endl; + } + + if (settings.NumberOfRetries_) { + freeText << "Number of retries: " << settings.NumberOfRetries_.GetRef() << Endl; + } + + row.FreeText(freeText); + } + + // export + TPrettyTable MakeTable(const NExport::TExportToS3Response&) { + return MakeTableS3(); + } + + void PrettyPrint(const NExport::TExportToS3Response& operation, TPrettyTable& table) { + PrettyPrintS3(operation, table); + } + + // import + TPrettyTable MakeTable(const NImport::TImportFromS3Response&) { + return MakeTableS3(); + } + + void PrettyPrint(const NImport::TImportFromS3Response& operation, TPrettyTable& table) { + PrettyPrintS3(operation, table); + } + + /// Index build TPrettyTable MakeTable(const NYdb::NTable::TBuildIndexOperation&) { return TPrettyTable({"id", "ready", "status", "state", "progress", "table", "index"}); } @@ -217,103 +217,103 @@ namespace { row.FreeText(freeText); } - // Common - template <typename T> - void PrintOperationImpl(const T& operation, EOutputFormat format) { - switch (format) { + // Common + template <typename T> + void PrintOperationImpl(const T& operation, EOutputFormat format) { + switch (format) { case EOutputFormat::Default: case EOutputFormat::Pretty: - { - auto table = MakeTable(operation); - PrettyPrint(operation, table); - Cout << table << Endl; - break; - } - - case EOutputFormat::Json: + { + auto table = MakeTable(operation); + PrettyPrint(operation, table); + Cout << table << Endl; + break; + } + + case EOutputFormat::Json: Cerr << "Warning! Option --json is deprecated and will be removed soon. " << "Use \"--format proto-json-base64\" option instead." << Endl; [[fallthrough]]; case EOutputFormat::ProtoJsonBase64: - Cout << operation.ToJsonString() << Endl; - break; - - default: - Y_FAIL("Unknown format"); - } - } - - template <typename T> - void PrintOperationsListImpl(const T& operations, EOutputFormat format) { - switch (format) { + Cout << operation.ToJsonString() << Endl; + break; + + default: + Y_FAIL("Unknown format"); + } + } + + template <typename T> + void PrintOperationsListImpl(const T& operations, EOutputFormat format) { + switch (format) { case EOutputFormat::Default: case EOutputFormat::Pretty: - if (operations.GetList()) { - auto table = MakeTable(operations.GetList().front()); - for (const auto& operation : operations.GetList()) { - PrettyPrint(operation, table); - } - Cout << table << Endl; - } - Cout << "Next page token: " << operations.NextPageToken() << Endl; - break; - - case EOutputFormat::Json: + if (operations.GetList()) { + auto table = MakeTable(operations.GetList().front()); + for (const auto& operation : operations.GetList()) { + PrettyPrint(operation, table); + } + Cout << table << Endl; + } + Cout << "Next page token: " << operations.NextPageToken() << Endl; + break; + + case EOutputFormat::Json: Cerr << "Warning! Option --json is deprecated and will be removed soon. " << "Use \"--format proto-json-base64\" option instead." << Endl; [[fallthrough]]; case EOutputFormat::ProtoJsonBase64: - Cout << operations.ToJsonString() << Endl; - break; - - default: - Y_FAIL("Unknown format"); - } - } - -} - + Cout << operations.ToJsonString() << Endl; + break; + + default: + Y_FAIL("Unknown format"); + } + } + +} + /// Common void PrintOperation(const TOperation& operation, EOutputFormat format) { PrintOperationImpl(operation, format); } -/// YT -void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format) { - PrintOperationImpl(operation, format); -} - -void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToYtResponse>& operations, EOutputFormat format) { - PrintOperationsListImpl(operations, format); -} - -/// S3 -// export -void PrintOperation(const NExport::TExportToS3Response& operation, EOutputFormat format) { - PrintOperationImpl(operation, format); -} - -void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToS3Response>& operations, EOutputFormat format) { - PrintOperationsListImpl(operations, format); -} - -// import -void PrintOperation(const NImport::TImportFromS3Response& operation, EOutputFormat format) { - PrintOperationImpl(operation, format); -} - -void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS3Response>& operations, EOutputFormat format) { - PrintOperationsListImpl(operations, format); -} - -/// Index build +/// YT +void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format) { + PrintOperationImpl(operation, format); +} + +void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToYtResponse>& operations, EOutputFormat format) { + PrintOperationsListImpl(operations, format); +} + +/// S3 +// export +void PrintOperation(const NExport::TExportToS3Response& operation, EOutputFormat format) { + PrintOperationImpl(operation, format); +} + +void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToS3Response>& operations, EOutputFormat format) { + PrintOperationsListImpl(operations, format); +} + +// import +void PrintOperation(const NImport::TImportFromS3Response& operation, EOutputFormat format) { + PrintOperationImpl(operation, format); +} + +void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS3Response>& operations, EOutputFormat format) { + PrintOperationsListImpl(operations, format); +} + +/// Index build void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format) { PrintOperationImpl(operation, format); -} +} void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format) { PrintOperationsListImpl(operations, format); -} +} } } diff --git a/ydb/public/lib/ydb_cli/common/print_operation.h b/ydb/public/lib/ydb_cli/common/print_operation.h index abfc826fa63..95e359ce03d 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.h +++ b/ydb/public/lib/ydb_cli/common/print_operation.h @@ -1,32 +1,32 @@ -#pragma once - -#include "formats.h" +#pragma once + +#include "formats.h" #include <ydb/public/sdk/cpp/client/ydb_export/export.h> #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -namespace NYdb { -namespace NConsoleClient { - + +namespace NYdb { +namespace NConsoleClient { + /// Common void PrintOperation(const TOperation& operation, EOutputFormat format); -/// YT -void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format); -void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToYtResponse>& operations, EOutputFormat format); - -/// S3 -// export -void PrintOperation(const NExport::TExportToS3Response& operation, EOutputFormat format); -void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToS3Response>& operations, EOutputFormat format); -// import -void PrintOperation(const NImport::TImportFromS3Response& operation, EOutputFormat format); -void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS3Response>& operations, EOutputFormat format); - +/// YT +void PrintOperation(const NExport::TExportToYtResponse& operation, EOutputFormat format); +void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToYtResponse>& operations, EOutputFormat format); + +/// S3 +// export +void PrintOperation(const NExport::TExportToS3Response& operation, EOutputFormat format); +void PrintOperationsList(const NOperation::TOperationsList<NExport::TExportToS3Response>& operations, EOutputFormat format); +// import +void PrintOperation(const NImport::TImportFromS3Response& operation, EOutputFormat format); +void PrintOperationsList(const NOperation::TOperationsList<NImport::TImportFromS3Response>& operations, EOutputFormat format); + /// Index build void PrintOperation(const NYdb::NTable::TBuildIndexOperation& operation, EOutputFormat format); void PrintOperationsList(const NOperation::TOperationsList<NYdb::NTable::TBuildIndexOperation>& operations, EOutputFormat format); -} -} +} +} diff --git a/ydb/public/lib/ydb_cli/common/print_utils.cpp b/ydb/public/lib/ydb_cli/common/print_utils.cpp index 5adde515f30..2cb520e578b 100644 --- a/ydb/public/lib/ydb_cli/common/print_utils.cpp +++ b/ydb/public/lib/ydb_cli/common/print_utils.cpp @@ -1,65 +1,65 @@ -#include "print_utils.h" - -#include <util/string/printf.h> - -namespace NYdb { -namespace NConsoleClient { - -void PrintSchemeEntry(IOutputStream& o, const NScheme::TSchemeEntry& entry, NColorizer::TColors colors) { - switch (entry.Type) { - case NScheme::ESchemeEntryType::Directory: - o << colors.LightBlueColor(); - break; - case NScheme::ESchemeEntryType::Table: - o << colors.WhiteColor(); - break; - case NScheme::ESchemeEntryType::PqGroup: - o << colors.BrownColor(); - break; - case NScheme::ESchemeEntryType::SubDomain: - o << colors.CyanColor(); - break; - case NScheme::ESchemeEntryType::RtmrVolume: - o << colors.LightGreenColor(); - break; - case NScheme::ESchemeEntryType::BlockStoreVolume: - o << colors.LightPurpleColor(); - break; - case NScheme::ESchemeEntryType::CoordinationNode: - o << colors.YellowColor(); - break; - default: - o << colors.RedColor(); - } - o << entry.Name << colors.OldColor(); -} - -TString PrettySize(size_t size) { - double sizeFormat = size; - TString mod = "b"; - const char* mods[] = { "Kb", "Mb", "Gb", "Tb", "Pb", "Eb" }; - TString numFormat = "%.0f"; - - for (const char* nextMod : mods) { - if (sizeFormat > 1024) { - sizeFormat /= 1024; - mod = nextMod; - numFormat = "%.02f"; - } else { - break; - } - } - - return Sprintf((numFormat + " %s").data(), sizeFormat, mod.data()); -} - -TString FormatTime(TInstant time) { - if (time.GetValue()) { - return time.ToRfc822StringLocal(); - } else { - return "Unknown"; - } -}; - -} -} +#include "print_utils.h" + +#include <util/string/printf.h> + +namespace NYdb { +namespace NConsoleClient { + +void PrintSchemeEntry(IOutputStream& o, const NScheme::TSchemeEntry& entry, NColorizer::TColors colors) { + switch (entry.Type) { + case NScheme::ESchemeEntryType::Directory: + o << colors.LightBlueColor(); + break; + case NScheme::ESchemeEntryType::Table: + o << colors.WhiteColor(); + break; + case NScheme::ESchemeEntryType::PqGroup: + o << colors.BrownColor(); + break; + case NScheme::ESchemeEntryType::SubDomain: + o << colors.CyanColor(); + break; + case NScheme::ESchemeEntryType::RtmrVolume: + o << colors.LightGreenColor(); + break; + case NScheme::ESchemeEntryType::BlockStoreVolume: + o << colors.LightPurpleColor(); + break; + case NScheme::ESchemeEntryType::CoordinationNode: + o << colors.YellowColor(); + break; + default: + o << colors.RedColor(); + } + o << entry.Name << colors.OldColor(); +} + +TString PrettySize(size_t size) { + double sizeFormat = size; + TString mod = "b"; + const char* mods[] = { "Kb", "Mb", "Gb", "Tb", "Pb", "Eb" }; + TString numFormat = "%.0f"; + + for (const char* nextMod : mods) { + if (sizeFormat > 1024) { + sizeFormat /= 1024; + mod = nextMod; + numFormat = "%.02f"; + } else { + break; + } + } + + return Sprintf((numFormat + " %s").data(), sizeFormat, mod.data()); +} + +TString FormatTime(TInstant time) { + if (time.GetValue()) { + return time.ToRfc822StringLocal(); + } else { + return "Unknown"; + } +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/print_utils.h b/ydb/public/lib/ydb_cli/common/print_utils.h index 379a2c68f96..8b7e8755c14 100644 --- a/ydb/public/lib/ydb_cli/common/print_utils.h +++ b/ydb/public/lib/ydb_cli/common/print_utils.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <library/cpp/colorizer/colors.h> - -namespace NYdb { -namespace NConsoleClient { - -void PrintSchemeEntry(IOutputStream& o, const NScheme::TSchemeEntry& entry, NColorizer::TColors colors); -TString FormatTime(TInstant time); -TString PrettySize(size_t size); - -} -} + +namespace NYdb { +namespace NConsoleClient { + +void PrintSchemeEntry(IOutputStream& o, const NScheme::TSchemeEntry& entry, NColorizer::TColors colors); +TString FormatTime(TInstant time); +TString PrettySize(size_t size); + +} +} diff --git a/ydb/public/lib/ydb_cli/common/recursive_list.cpp b/ydb/public/lib/ydb_cli/common/recursive_list.cpp index 2f020b41a80..3decf4fb56c 100644 --- a/ydb/public/lib/ydb_cli/common/recursive_list.cpp +++ b/ydb/public/lib/ydb_cli/common/recursive_list.cpp @@ -1,68 +1,68 @@ -#include "recursive_list.h" - -#include <util/string/join.h> - -namespace NYdb { -namespace NConsoleClient { - -using namespace NScheme; - -namespace { - - TSchemeEntry ReplaceNameWithPath(const TSchemeEntry& entry, const TString& path) { - TSchemeEntry result(entry); - result.Name = path; - return result; - } - +#include "recursive_list.h" + +#include <util/string/join.h> + +namespace NYdb { +namespace NConsoleClient { + +using namespace NScheme; + +namespace { + + TSchemeEntry ReplaceNameWithPath(const TSchemeEntry& entry, const TString& path) { + TSchemeEntry result(entry); + result.Name = path; + return result; + } + TStatus RecursiveList(TVector<TSchemeEntry>& dst, TSchemeClient& client, - const TString& path, const TRecursiveListSettings& settings, bool addSelf = false) { - - auto list = client.ListDirectory(path, settings.ListDirectorySettings_).GetValueSync(); - if (!list.IsSuccess()) { - return list; - } - - const auto& self = list.GetEntry(); + const TString& path, const TRecursiveListSettings& settings, bool addSelf = false) { + + auto list = client.ListDirectory(path, settings.ListDirectorySettings_).GetValueSync(); + if (!list.IsSuccess()) { + return list; + } + + const auto& self = list.GetEntry(); if (addSelf && settings.Filter_(self)) { - dst.push_back(ReplaceNameWithPath(self, path)); - } - - for (const auto& child : list.GetChildren()) { - if (settings.SkipSys_ && child.Name.StartsWith(".sys")) { - continue; - } - - if (settings.Filter_(child)) { - dst.push_back(ReplaceNameWithPath(child, Join('/', path, child.Name))); - } - - switch (child.Type) { - case ESchemeEntryType::SubDomain: - case ESchemeEntryType::Directory: { - auto status = RecursiveList(dst, client, Join('/', path, child.Name), settings); - if (!status.IsSuccess()) { - return status; - } - break; - } - default: - break; - } - } - - return list; - } - -} // anonymous - + dst.push_back(ReplaceNameWithPath(self, path)); + } + + for (const auto& child : list.GetChildren()) { + if (settings.SkipSys_ && child.Name.StartsWith(".sys")) { + continue; + } + + if (settings.Filter_(child)) { + dst.push_back(ReplaceNameWithPath(child, Join('/', path, child.Name))); + } + + switch (child.Type) { + case ESchemeEntryType::SubDomain: + case ESchemeEntryType::Directory: { + auto status = RecursiveList(dst, client, Join('/', path, child.Name), settings); + if (!status.IsSuccess()) { + return status; + } + break; + } + default: + break; + } + } + + return list; + } + +} // anonymous + TRecursiveListResult RecursiveList(TSchemeClient& client, const TString& path, - const TRecursiveListSettings& settings, bool addSelf) { - - TVector<TSchemeEntry> entries; + const TRecursiveListSettings& settings, bool addSelf) { + + TVector<TSchemeEntry> entries; auto status = RecursiveList(entries, client, path, settings, addSelf); - return {entries, status}; -} - -} -} + return {entries, status}; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/recursive_list.h b/ydb/public/lib/ydb_cli/common/recursive_list.h index ae026faf8af..3448c00f9b7 100644 --- a/ydb/public/lib/ydb_cli/common/recursive_list.h +++ b/ydb/public/lib/ydb_cli/common/recursive_list.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> - -#include <util/generic/vector.h> - -namespace NYdb { -namespace NConsoleClient { - -struct TRecursiveListSettings { - using TSelf = TRecursiveListSettings; - using TFilterOp = std::function<bool(const NScheme::TSchemeEntry&)>; - - FLUENT_SETTING_DEFAULT(NScheme::TListDirectorySettings, ListDirectorySettings, {}); - FLUENT_SETTING_DEFAULT(bool, SkipSys, true); - FLUENT_SETTING_DEFAULT(TFilterOp, Filter, &NopFilter); - -private: - static inline bool NopFilter(const NScheme::TSchemeEntry&) { - return true; - } -}; - -struct TRecursiveListResult { - TVector<NScheme::TSchemeEntry> Entries; // name of the entry contains full path - TStatus Status; // last successful or first unsuccessful status -}; - -TRecursiveListResult RecursiveList(NScheme::TSchemeClient& client, const TString& path, + +#include <util/generic/vector.h> + +namespace NYdb { +namespace NConsoleClient { + +struct TRecursiveListSettings { + using TSelf = TRecursiveListSettings; + using TFilterOp = std::function<bool(const NScheme::TSchemeEntry&)>; + + FLUENT_SETTING_DEFAULT(NScheme::TListDirectorySettings, ListDirectorySettings, {}); + FLUENT_SETTING_DEFAULT(bool, SkipSys, true); + FLUENT_SETTING_DEFAULT(TFilterOp, Filter, &NopFilter); + +private: + static inline bool NopFilter(const NScheme::TSchemeEntry&) { + return true; + } +}; + +struct TRecursiveListResult { + TVector<NScheme::TSchemeEntry> Entries; // name of the entry contains full path + TStatus Status; // last successful or first unsuccessful status +}; + +TRecursiveListResult RecursiveList(NScheme::TSchemeClient& client, const TString& path, const TRecursiveListSettings& settings = {}, bool addSelf = true); - -} -} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp index ec31f58f244..f5371d22a28 100644 --- a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp +++ b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp @@ -1,7 +1,7 @@ #include "tabbed_table.h" #include "common.h" -#include "print_utils.h" +#include "print_utils.h" namespace NYdb { namespace NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/common/ya.make b/ydb/public/lib/ydb_cli/common/ya.make index a77c61042c9..35523bfb4cd 100644 --- a/ydb/public/lib/ydb_cli/common/ya.make +++ b/ydb/public/lib/ydb_cli/common/ya.make @@ -3,24 +3,24 @@ LIBRARY(common) OWNER(g:kikimr) SRCS( - aws.cpp + aws.cpp command.cpp common.cpp examples.cpp - format.cpp + format.cpp interruptible.cpp normalize_path.cpp parameters.cpp - pretty_table.cpp - print_operation.cpp - print_utils.cpp + pretty_table.cpp + print_operation.cpp + print_utils.cpp profile_manager.cpp query_stats.cpp - recursive_list.cpp + recursive_list.cpp root.cpp tabbed_table.cpp ydb_updater.cpp - yt.cpp + yt.cpp ) PEERDIR( @@ -36,8 +36,8 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_types/credentials ) -GENERATE_ENUM_SERIALIZATION(formats.h) - +GENERATE_ENUM_SERIALIZATION(formats.h) + END() RECURSE_FOR_TESTS( diff --git a/ydb/public/lib/ydb_cli/common/yt.cpp b/ydb/public/lib/ydb_cli/common/yt.cpp index a3805a63a98..91c37eaec22 100644 --- a/ydb/public/lib/ydb_cli/common/yt.cpp +++ b/ydb/public/lib/ydb_cli/common/yt.cpp @@ -1,41 +1,41 @@ -#include "yt.h" - -#include <util/folder/dirut.h> -#include <util/folder/path.h> -#include <util/generic/maybe.h> -#include <util/stream/file.h> -#include <util/string/split.h> -#include <util/string/strip.h> - -namespace NYdb { -namespace NConsoleClient { - -void TCommandWithYtProxy::ParseYtProxy(const TString& proxy) { - if (proxy.empty()) { - throw TMissUseException() << "No YT proxy provided."; - } - - TMaybe<ui16> port; - - try { - Split(proxy, ':', YtHost, port); - } catch (const yexception& ex) { - throw TMissUseException() << "Bad YT proxy format: \"" << proxy << "\"."; - } - - if (YtHost.find('.') == TString::npos && YtHost.find("localhost") == TString::npos) { - YtHost.append(".yt.yandex.net"); - } - - YtPort = port.GetOrElse(DefaultYtPort); -} - -const TString TCommandWithYtToken::YtTokenFile = "~/.yt/token"; - -void TCommandWithYtToken::ReadYtToken() { - TString tokenFile = YtTokenFile; +#include "yt.h" + +#include <util/folder/dirut.h> +#include <util/folder/path.h> +#include <util/generic/maybe.h> +#include <util/stream/file.h> +#include <util/string/split.h> +#include <util/string/strip.h> + +namespace NYdb { +namespace NConsoleClient { + +void TCommandWithYtProxy::ParseYtProxy(const TString& proxy) { + if (proxy.empty()) { + throw TMissUseException() << "No YT proxy provided."; + } + + TMaybe<ui16> port; + + try { + Split(proxy, ':', YtHost, port); + } catch (const yexception& ex) { + throw TMissUseException() << "Bad YT proxy format: \"" << proxy << "\"."; + } + + if (YtHost.find('.') == TString::npos && YtHost.find("localhost") == TString::npos) { + YtHost.append(".yt.yandex.net"); + } + + YtPort = port.GetOrElse(DefaultYtPort); +} + +const TString TCommandWithYtToken::YtTokenFile = "~/.yt/token"; + +void TCommandWithYtToken::ReadYtToken() { + TString tokenFile = YtTokenFile; YtToken = ReadFromFile(tokenFile, "YT token"); -} - -} -} +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/yt.h b/ydb/public/lib/ydb_cli/common/yt.h index a6d167dc937..723d354b501 100644 --- a/ydb/public/lib/ydb_cli/common/yt.h +++ b/ydb/public/lib/ydb_cli/common/yt.h @@ -1,56 +1,56 @@ -#pragma once - -#include "command.h" - -#include <util/system/env.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandWithYtProxy { -protected: - template <typename TOpt> - void ParseYtProxy(const TClientCommand::TConfig& config, const TOpt opt) { - if (config.ParseResult->Has(opt)) { - ParseYtProxy(config.ParseResult->Get(opt)); - } else { - ParseYtProxy(GetEnv("YT_PROXY")); - } - } - - TString YtHost; - ui16 YtPort; - -private: - void ParseYtProxy(const TString& proxy); - - static constexpr ui16 DefaultYtPort = 80; -}; - -class TCommandWithYtToken { -protected: - template <typename TOpt> - void ParseYtToken(const TClientCommand::TConfig& config, const TOpt opt) { - if (config.ParseResult->Has(opt)) { - YtToken = config.ParseResult->Get(opt); - } else { - TString token = GetEnv("YT_TOKEN"); - - if (!token.empty()) { - YtToken = std::move(token); - } else { - ReadYtToken(); - } - } - } - - TString YtToken; - - static const TString YtTokenFile; - -private: - void ReadYtToken(); -}; - -} -} +#pragma once + +#include "command.h" + +#include <util/system/env.h> + +namespace NYdb { +namespace NConsoleClient { + +class TCommandWithYtProxy { +protected: + template <typename TOpt> + void ParseYtProxy(const TClientCommand::TConfig& config, const TOpt opt) { + if (config.ParseResult->Has(opt)) { + ParseYtProxy(config.ParseResult->Get(opt)); + } else { + ParseYtProxy(GetEnv("YT_PROXY")); + } + } + + TString YtHost; + ui16 YtPort; + +private: + void ParseYtProxy(const TString& proxy); + + static constexpr ui16 DefaultYtPort = 80; +}; + +class TCommandWithYtToken { +protected: + template <typename TOpt> + void ParseYtToken(const TClientCommand::TConfig& config, const TOpt opt) { + if (config.ParseResult->Has(opt)) { + YtToken = config.ParseResult->Get(opt); + } else { + TString token = GetEnv("YT_TOKEN"); + + if (!token.empty()) { + YtToken = std::move(token); + } else { + ReadYtToken(); + } + } + } + + TString YtToken; + + static const TString YtTokenFile; + +private: + void ReadYtToken(); +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/dump/dump.cpp b/ydb/public/lib/ydb_cli/dump/dump.cpp index 6a3a1c087df..093d90b54c2 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.cpp +++ b/ydb/public/lib/ydb_cli/dump/dump.cpp @@ -1,75 +1,75 @@ -#include "dump.h" -#include "dump_impl.h" -#include "restore_impl.h" - +#include "dump.h" +#include "dump_impl.h" +#include "restore_impl.h" + #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <util/string/printf.h> - -namespace NYdb { -namespace NDump { - -extern const char SCHEME_FILE_NAME[] = "scheme.pb"; -extern const char INCOMPLETE_FILE_NAME[] = "incomplete"; -extern const char EMPTY_FILE_NAME[] = "empty_dir"; - -TString DataFileName(ui32 id) { - return Sprintf("data_%02d.csv", id); -} - -class TClient::TImpl { -public: - explicit TImpl(const TDriver& driver) - : ImportClient(driver) - , OperationClient(driver) - , SchemeClient(driver) - , TableClient(driver) - { - } - - TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { - auto client = TDumpClient(SchemeClient, TableClient); - return client.Dump(dbPath, fsPath, settings); - } - - TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { + +#include <util/string/printf.h> + +namespace NYdb { +namespace NDump { + +extern const char SCHEME_FILE_NAME[] = "scheme.pb"; +extern const char INCOMPLETE_FILE_NAME[] = "incomplete"; +extern const char EMPTY_FILE_NAME[] = "empty_dir"; + +TString DataFileName(ui32 id) { + return Sprintf("data_%02d.csv", id); +} + +class TClient::TImpl { +public: + explicit TImpl(const TDriver& driver) + : ImportClient(driver) + , OperationClient(driver) + , SchemeClient(driver) + , TableClient(driver) + { + } + + TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { + auto client = TDumpClient(SchemeClient, TableClient); + return client.Dump(dbPath, fsPath, settings); + } + + TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { auto client = TRestoreClient(ImportClient, OperationClient, SchemeClient, TableClient); - return client.Restore(fsPath, dbPath, settings); - } - -private: - NImport::TImportClient ImportClient; - NOperation::TOperationClient OperationClient; - NScheme::TSchemeClient SchemeClient; - NTable::TTableClient TableClient; - -}; // TImpl - -TDumpResult::TDumpResult(TStatus&& status) - : TStatus(std::move(status)) -{ -} - -TRestoreResult::TRestoreResult(TStatus&& status) - : TStatus(std::move(status)) -{ -} - -TClient::TClient(const TDriver& driver) - : Impl_(new TImpl(driver)) -{ -} - -TDumpResult TClient::Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { - return Impl_->Dump(dbPath, fsPath, settings); -} - -TRestoreResult TClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { - return Impl_->Restore(fsPath, dbPath, settings); -} - -} // NDump -} // NYdb + return client.Restore(fsPath, dbPath, settings); + } + +private: + NImport::TImportClient ImportClient; + NOperation::TOperationClient OperationClient; + NScheme::TSchemeClient SchemeClient; + NTable::TTableClient TableClient; + +}; // TImpl + +TDumpResult::TDumpResult(TStatus&& status) + : TStatus(std::move(status)) +{ +} + +TRestoreResult::TRestoreResult(TStatus&& status) + : TStatus(std::move(status)) +{ +} + +TClient::TClient(const TDriver& driver) + : Impl_(new TImpl(driver)) +{ +} + +TDumpResult TClient::Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { + return Impl_->Dump(dbPath, fsPath, settings); +} + +TRestoreResult TClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { + return Impl_->Restore(fsPath, dbPath, settings); +} + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/dump.h b/ydb/public/lib/ydb_cli/dump/dump.h index 118d1f66455..877948d1056 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.h +++ b/ydb/public/lib/ydb_cli/dump/dump.h @@ -1,103 +1,103 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> #include <ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -#include <util/generic/size_literals.h> - -namespace NYdb { -namespace NDump { - -extern const char SCHEME_FILE_NAME[10]; -extern const char INCOMPLETE_FILE_NAME[11]; -extern const char EMPTY_FILE_NAME[10]; - -TString DataFileName(ui32 id); - -/// dump -struct TDumpSettings: public TOperationRequestSettings<TDumpSettings> { - using TSelf = TDumpSettings; - -}; // TDumpSettings - -class TDumpResult: public TStatus { -public: - TDumpResult(TStatus&& status); - -}; // TDumpResult - -/// restore -struct TRateLimiterSettings { - using TSelf = TRateLimiterSettings; - - FLUENT_SETTING_DEFAULT(ui32, Rate, 30); - FLUENT_SETTING_DEFAULT(TDuration, Interval, TDuration::Seconds(1)); - FLUENT_SETTING_DEFAULT(TDuration, ReactionTime, TDuration::MilliSeconds(50)); - - TSelf& WithBandwidth(ui64 bandwidth, ui64 batchSize) { - return Rate(Max<ui64>(1, (Interval_.Seconds() * bandwidth + batchSize - 1) / batchSize)); - } - - TSelf& WithRps(ui64 maxUploadRps) { - return Rate(Max<ui64>(1, maxUploadRps * Interval_.Seconds())); - } - - ui32 GetRps() const { + +#include <util/generic/size_literals.h> + +namespace NYdb { +namespace NDump { + +extern const char SCHEME_FILE_NAME[10]; +extern const char INCOMPLETE_FILE_NAME[11]; +extern const char EMPTY_FILE_NAME[10]; + +TString DataFileName(ui32 id); + +/// dump +struct TDumpSettings: public TOperationRequestSettings<TDumpSettings> { + using TSelf = TDumpSettings; + +}; // TDumpSettings + +class TDumpResult: public TStatus { +public: + TDumpResult(TStatus&& status); + +}; // TDumpResult + +/// restore +struct TRateLimiterSettings { + using TSelf = TRateLimiterSettings; + + FLUENT_SETTING_DEFAULT(ui32, Rate, 30); + FLUENT_SETTING_DEFAULT(TDuration, Interval, TDuration::Seconds(1)); + FLUENT_SETTING_DEFAULT(TDuration, ReactionTime, TDuration::MilliSeconds(50)); + + TSelf& WithBandwidth(ui64 bandwidth, ui64 batchSize) { + return Rate(Max<ui64>(1, (Interval_.Seconds() * bandwidth + batchSize - 1) / batchSize)); + } + + TSelf& WithRps(ui64 maxUploadRps) { + return Rate(Max<ui64>(1, maxUploadRps * Interval_.Seconds())); + } + + ui32 GetRps() const { return Max<ui32>(1, Rate_ * TDuration::Seconds(1).MilliSeconds() / Interval_.MilliSeconds()); - } - -}; // TRateLimiterSettings - -struct TRestoreSettings: public TOperationRequestSettings<TRestoreSettings> { - using TSelf = TRestoreSettings; - - enum class EMode { - Yql, - BulkUpsert, - ImportData, - }; - - static constexpr ui64 MaxBytesPerRequest = 8_MB; - - FLUENT_SETTING_DEFAULT(EMode, Mode, EMode::Yql); - FLUENT_SETTING_DEFAULT(bool, DryRun, false); - FLUENT_SETTING_DEFAULT(bool, RestoreData, true); - FLUENT_SETTING_DEFAULT(bool, RestoreIndexes, true); - FLUENT_SETTING_DEFAULT(bool, SkipDocumentTables, false); + } + +}; // TRateLimiterSettings + +struct TRestoreSettings: public TOperationRequestSettings<TRestoreSettings> { + using TSelf = TRestoreSettings; + + enum class EMode { + Yql, + BulkUpsert, + ImportData, + }; + + static constexpr ui64 MaxBytesPerRequest = 8_MB; + + FLUENT_SETTING_DEFAULT(EMode, Mode, EMode::Yql); + FLUENT_SETTING_DEFAULT(bool, DryRun, false); + FLUENT_SETTING_DEFAULT(bool, RestoreData, true); + FLUENT_SETTING_DEFAULT(bool, RestoreIndexes, true); + FLUENT_SETTING_DEFAULT(bool, SkipDocumentTables, false); FLUENT_SETTING_DEFAULT(bool, SavePartialResult, false); - - FLUENT_SETTING_DEFAULT(ui64, MemLimit, 16_MB); - FLUENT_SETTING_DEFAULT(ui64, RowsPerRequest, 0); - FLUENT_SETTING_DEFAULT(ui64, BytesPerRequest, 512_KB); - FLUENT_SETTING_DEFAULT(ui64, RequestUnitsPerRequest, 30); - FLUENT_SETTING_DEFAULT(ui64, FileBufferSize, 2_MB); - FLUENT_SETTING_DEFAULT(ui32, InFly, 10); - FLUENT_SETTING_DEFAULT(TRateLimiterSettings, RateLimiterSettings, {}); - -}; // TRestoreSettings - -class TRestoreResult: public TStatus { -public: - TRestoreResult(TStatus&& status); - -}; // TRestoreResult - -/// client -class TClient { -private: - class TImpl; - -public: - explicit TClient(const TDriver& driver); - - TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); - TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {}); - -private: - std::shared_ptr<TImpl> Impl_; - -}; // TClient - -} // NDump -} // NYdb + + FLUENT_SETTING_DEFAULT(ui64, MemLimit, 16_MB); + FLUENT_SETTING_DEFAULT(ui64, RowsPerRequest, 0); + FLUENT_SETTING_DEFAULT(ui64, BytesPerRequest, 512_KB); + FLUENT_SETTING_DEFAULT(ui64, RequestUnitsPerRequest, 30); + FLUENT_SETTING_DEFAULT(ui64, FileBufferSize, 2_MB); + FLUENT_SETTING_DEFAULT(ui32, InFly, 10); + FLUENT_SETTING_DEFAULT(TRateLimiterSettings, RateLimiterSettings, {}); + +}; // TRestoreSettings + +class TRestoreResult: public TStatus { +public: + TRestoreResult(TStatus&& status); + +}; // TRestoreResult + +/// client +class TClient { +private: + class TImpl; + +public: + explicit TClient(const TDriver& driver); + + TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); + TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {}); + +private: + std::shared_ptr<TImpl> Impl_; + +}; // TClient + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp index 8b09e5f66dd..a02f7facae5 100644 --- a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp +++ b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp @@ -1,25 +1,25 @@ -#include "dump_impl.h" - +#include "dump_impl.h" + #include <ydb/public/lib/ydb_cli/dump/util/util.h> -namespace NYdb { -namespace NDump { - -TDumpClient::TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient) - : SchemeClient(schemeClient) - , TableClient(tableClient) -{ - Y_UNUSED(SchemeClient); - Y_UNUSED(TableClient); -} - -TDumpResult TDumpClient::Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { - Y_UNUSED(dbPath); - Y_UNUSED(TableClient); - Y_UNUSED(fsPath); - Y_UNUSED(settings); - return Result<TDumpResult>(EStatus::CLIENT_CALL_UNIMPLEMENTED); -} - -} // NDump -} // NYdb +namespace NYdb { +namespace NDump { + +TDumpClient::TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient) + : SchemeClient(schemeClient) + , TableClient(tableClient) +{ + Y_UNUSED(SchemeClient); + Y_UNUSED(TableClient); +} + +TDumpResult TDumpClient::Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) { + Y_UNUSED(dbPath); + Y_UNUSED(TableClient); + Y_UNUSED(fsPath); + Y_UNUSED(settings); + return Result<TDumpResult>(EStatus::CLIENT_CALL_UNIMPLEMENTED); +} + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.h b/ydb/public/lib/ydb_cli/dump/dump_impl.h index 7dce3a01235..ac01afbe613 100644 --- a/ydb/public/lib/ydb_cli/dump/dump_impl.h +++ b/ydb/public/lib/ydb_cli/dump/dump_impl.h @@ -1,24 +1,24 @@ -#pragma once - -#include "dump.h" - +#pragma once + +#include "dump.h" + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -namespace NYdb { -namespace NDump { - -class TDumpClient { -public: - explicit TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient); - - TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); - -private: - NScheme::TSchemeClient& SchemeClient; - NTable::TTableClient& TableClient; - -}; // TDumpClient - -} // NDump -} // NYdb + +namespace NYdb { +namespace NDump { + +class TDumpClient { +public: + explicit TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient); + + TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); + +private: + NScheme::TSchemeClient& SchemeClient; + NTable::TTableClient& TableClient; + +}; // TDumpClient + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_compat.cpp b/ydb/public/lib/ydb_cli/dump/restore_compat.cpp index 37135a4f6aa..39984fd4210 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_compat.cpp +++ b/ydb/public/lib/ydb_cli/dump/restore_compat.cpp @@ -1,183 +1,183 @@ -#include "restore_compat.h" - +#include "restore_compat.h" + #include <ydb/library/backup/query_builder.h> #include <ydb/library/backup/query_uploader.h> - -namespace NYdb { -namespace NDump { - -using namespace NBackup; -using namespace NTable; - -namespace { - -class TDataWriter; - -class TDataAccumulator: public NPrivate::IDataAccumulator, protected TQueryBuilder { - friend class TDataWriter; - - ui64 RecordSize(const TString& line) const { - return line.size(); - } - - ui64 RecordRequestUnitsX2(const TString& line) const { - const ui64 kbytes = (RecordSize(line) + 1_KB - 1) / 1_KB; - return UseBulkUpsert ? kbytes : 4 * kbytes; - } - - static ui64 CalcRequestUnits(ui64 requestUnitsX2) { - return (requestUnitsX2 + 1) / 2; - } - -public: - explicit TDataAccumulator( - const TString& path, - const TTableDescription& desc, - const TRestoreSettings& settings) - : TQueryBuilder(path, desc.GetColumns()) - , UseBulkUpsert(settings.Mode_ == TRestoreSettings::EMode::BulkUpsert) - , RowsPerRequest(settings.RowsPerRequest_) - , BytesPerRequest(settings.BytesPerRequest_) - , RequestUnitsPerRequest(settings.RequestUnitsPerRequest_) - , Rows(0) - , Bytes(0) - , RequestUnitsX2(0) - { - Begin(); - } - - bool Fits(const TString& line) const override { - if (RowsPerRequest > 0 && (Rows + 1) > RowsPerRequest) { - return Rows == 0; - } - - if (BytesPerRequest > 0 && (Bytes + RecordSize(line)) > BytesPerRequest) { - return Bytes == 0; - } - - if (RequestUnitsPerRequest > 0 && CalcRequestUnits(RequestUnitsX2 + RecordRequestUnitsX2(line)) > RequestUnitsPerRequest) { - return RequestUnitsX2 == 0; - } - - return true; - } - - void Feed(TString&& line) override { - AddLine(line); - - ++Rows; - Bytes += RecordSize(line); - RequestUnitsX2 += RecordRequestUnitsX2(line); - } - - bool Ready(bool force) const override { - if (RowsPerRequest > 0 && Rows >= RowsPerRequest) { - return true; - } - - if (BytesPerRequest > 0 && Bytes >= BytesPerRequest) { - return true; - } - - if (RequestUnitsPerRequest > 0 && CalcRequestUnits(RequestUnitsX2) >= RequestUnitsPerRequest) { - return true; - } - - if (force && (Rows || Bytes || RequestUnitsX2)) { - return true; - } - - return false; - } - - TString GetData(bool) override { - Rows = 0; - Bytes = 0; - RequestUnitsX2 = 0; - - return {}; // Writer gets data directly from accumulator - } - -private: - const bool UseBulkUpsert; - const ui64 RowsPerRequest; // 0 for inf - const ui64 BytesPerRequest; // 0 for inf - const ui64 RequestUnitsPerRequest; // 0 for inf - - ui64 Rows; - ui64 Bytes; - ui64 RequestUnitsX2; - -}; // TDataAccumulator - -class TDataWriter: public NPrivate::IDataWriter { -public: - explicit TDataWriter( - const TString& path, - TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator, - const TRestoreSettings& settings) - : Path(path) - , TableClient(tableClient) - , Accumulator(dynamic_cast<TDataAccumulator*>(accumulator)) - , UseBulkUpsert(settings.Mode_ == TRestoreSettings::EMode::BulkUpsert) - { - Y_ENSURE(Accumulator); - - TUploader::TOptions opts; - opts.InFly = settings.InFly_; - opts.Rate = settings.RateLimiterSettings_.Rate_; - opts.Interval = settings.RateLimiterSettings_.Interval_; - opts.ReactionTime = settings.RateLimiterSettings_.ReactionTime_; - - Uploader = MakeHolder<TUploader>(opts, TableClient, Accumulator->GetQueryString()); - } - - bool Push(TString&&) override { - bool ok; - - if (UseBulkUpsert) { - ok = Uploader->Push(Path, Accumulator->EndAndGetResultingValue()); - } else { - ok = Uploader->Push(Accumulator->EndAndGetResultingParams()); - } - - if (ok) { - Accumulator->Begin(); - } - - return ok; - } - - void Wait() override { - Uploader->WaitAllJobs(); - } - -private: - const TString Path; - TTableClient& TableClient; - TDataAccumulator* Accumulator; - const bool UseBulkUpsert; - THolder<TUploader> Uploader; - -}; // TDataWriter - -} // anonymous - -NPrivate::IDataAccumulator* CreateCompatAccumulator( - const TString& path, - const TTableDescription& desc, - const TRestoreSettings& settings) { - return new TDataAccumulator(path, desc, settings); -} - -NPrivate::IDataWriter* CreateCompatWriter( - const TString& path, - TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator, - const TRestoreSettings& settings) { - return new TDataWriter(path, tableClient, accumulator, settings); -} - -} // NDump -} // NYdb + +namespace NYdb { +namespace NDump { + +using namespace NBackup; +using namespace NTable; + +namespace { + +class TDataWriter; + +class TDataAccumulator: public NPrivate::IDataAccumulator, protected TQueryBuilder { + friend class TDataWriter; + + ui64 RecordSize(const TString& line) const { + return line.size(); + } + + ui64 RecordRequestUnitsX2(const TString& line) const { + const ui64 kbytes = (RecordSize(line) + 1_KB - 1) / 1_KB; + return UseBulkUpsert ? kbytes : 4 * kbytes; + } + + static ui64 CalcRequestUnits(ui64 requestUnitsX2) { + return (requestUnitsX2 + 1) / 2; + } + +public: + explicit TDataAccumulator( + const TString& path, + const TTableDescription& desc, + const TRestoreSettings& settings) + : TQueryBuilder(path, desc.GetColumns()) + , UseBulkUpsert(settings.Mode_ == TRestoreSettings::EMode::BulkUpsert) + , RowsPerRequest(settings.RowsPerRequest_) + , BytesPerRequest(settings.BytesPerRequest_) + , RequestUnitsPerRequest(settings.RequestUnitsPerRequest_) + , Rows(0) + , Bytes(0) + , RequestUnitsX2(0) + { + Begin(); + } + + bool Fits(const TString& line) const override { + if (RowsPerRequest > 0 && (Rows + 1) > RowsPerRequest) { + return Rows == 0; + } + + if (BytesPerRequest > 0 && (Bytes + RecordSize(line)) > BytesPerRequest) { + return Bytes == 0; + } + + if (RequestUnitsPerRequest > 0 && CalcRequestUnits(RequestUnitsX2 + RecordRequestUnitsX2(line)) > RequestUnitsPerRequest) { + return RequestUnitsX2 == 0; + } + + return true; + } + + void Feed(TString&& line) override { + AddLine(line); + + ++Rows; + Bytes += RecordSize(line); + RequestUnitsX2 += RecordRequestUnitsX2(line); + } + + bool Ready(bool force) const override { + if (RowsPerRequest > 0 && Rows >= RowsPerRequest) { + return true; + } + + if (BytesPerRequest > 0 && Bytes >= BytesPerRequest) { + return true; + } + + if (RequestUnitsPerRequest > 0 && CalcRequestUnits(RequestUnitsX2) >= RequestUnitsPerRequest) { + return true; + } + + if (force && (Rows || Bytes || RequestUnitsX2)) { + return true; + } + + return false; + } + + TString GetData(bool) override { + Rows = 0; + Bytes = 0; + RequestUnitsX2 = 0; + + return {}; // Writer gets data directly from accumulator + } + +private: + const bool UseBulkUpsert; + const ui64 RowsPerRequest; // 0 for inf + const ui64 BytesPerRequest; // 0 for inf + const ui64 RequestUnitsPerRequest; // 0 for inf + + ui64 Rows; + ui64 Bytes; + ui64 RequestUnitsX2; + +}; // TDataAccumulator + +class TDataWriter: public NPrivate::IDataWriter { +public: + explicit TDataWriter( + const TString& path, + TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator, + const TRestoreSettings& settings) + : Path(path) + , TableClient(tableClient) + , Accumulator(dynamic_cast<TDataAccumulator*>(accumulator)) + , UseBulkUpsert(settings.Mode_ == TRestoreSettings::EMode::BulkUpsert) + { + Y_ENSURE(Accumulator); + + TUploader::TOptions opts; + opts.InFly = settings.InFly_; + opts.Rate = settings.RateLimiterSettings_.Rate_; + opts.Interval = settings.RateLimiterSettings_.Interval_; + opts.ReactionTime = settings.RateLimiterSettings_.ReactionTime_; + + Uploader = MakeHolder<TUploader>(opts, TableClient, Accumulator->GetQueryString()); + } + + bool Push(TString&&) override { + bool ok; + + if (UseBulkUpsert) { + ok = Uploader->Push(Path, Accumulator->EndAndGetResultingValue()); + } else { + ok = Uploader->Push(Accumulator->EndAndGetResultingParams()); + } + + if (ok) { + Accumulator->Begin(); + } + + return ok; + } + + void Wait() override { + Uploader->WaitAllJobs(); + } + +private: + const TString Path; + TTableClient& TableClient; + TDataAccumulator* Accumulator; + const bool UseBulkUpsert; + THolder<TUploader> Uploader; + +}; // TDataWriter + +} // anonymous + +NPrivate::IDataAccumulator* CreateCompatAccumulator( + const TString& path, + const TTableDescription& desc, + const TRestoreSettings& settings) { + return new TDataAccumulator(path, desc, settings); +} + +NPrivate::IDataWriter* CreateCompatWriter( + const TString& path, + TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator, + const TRestoreSettings& settings) { + return new TDataWriter(path, tableClient, accumulator, settings); +} + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_compat.h b/ydb/public/lib/ydb_cli/dump/restore_compat.h index db94ab4c728..c8a20d468d7 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_compat.h +++ b/ydb/public/lib/ydb_cli/dump/restore_compat.h @@ -1,20 +1,20 @@ -#pragma once - -#include "restore_impl.h" - -namespace NYdb { -namespace NDump { - -NPrivate::IDataAccumulator* CreateCompatAccumulator( - const TString& path, - const NTable::TTableDescription& desc, - const TRestoreSettings& settings); - -NPrivate::IDataWriter* CreateCompatWriter( - const TString& path, - NTable::TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator, - const TRestoreSettings& settings); - -} // NDump -} // NYdb +#pragma once + +#include "restore_impl.h" + +namespace NYdb { +namespace NDump { + +NPrivate::IDataAccumulator* CreateCompatAccumulator( + const TString& path, + const NTable::TTableDescription& desc, + const TRestoreSettings& settings); + +NPrivate::IDataWriter* CreateCompatWriter( + const TString& path, + NTable::TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator, + const TRestoreSettings& settings); + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp index 2f1b4e8e378..74f4ffd18c9 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp +++ b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp @@ -1,434 +1,434 @@ -#include "restore_impl.h" -#include "restore_import_data.h" -#include "restore_compat.h" - +#include "restore_impl.h" +#include "restore_import_data.h" +#include "restore_compat.h" + #include <ydb/public/api/protos/ydb_table.pb.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/lib/ydb_cli/common/recursive_list.h> #include <ydb/public/lib/ydb_cli/dump/util/util.h> - -#include <util/generic/hash.h> + +#include <util/generic/hash.h> #include <util/generic/hash_set.h> -#include <util/generic/maybe.h> -#include <util/generic/vector.h> -#include <util/stream/file.h> -#include <util/string/builder.h> -#include <util/string/join.h> - -namespace NYdb { -namespace NDump { - -using namespace NImport; -using namespace NOperation; +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/stream/file.h> +#include <util/string/builder.h> +#include <util/string/join.h> + +namespace NYdb { +namespace NDump { + +using namespace NImport; +using namespace NOperation; using namespace NScheme; -using namespace NTable; - -extern const char DOC_API_TABLE_VERSION_ATTR[] = "__document_api_version"; -extern const char DOC_API_REQUEST_TYPE[] = "_document_api_request"; - -namespace { - -bool IsFileExists(const TFsPath& path) { - return path.Exists() && path.IsFile(); -} - -Ydb::Table::CreateTableRequest ReadTableScheme(const TString& fsPath) { - Ydb::Table::CreateTableRequest proto; - Y_ENSURE(google::protobuf::TextFormat::ParseFromString(TFileInput(fsPath).ReadAll(), &proto)); - return proto; -} - -TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) { - return TProtoAccessor::FromProto(proto); -} - -bool HasRunningIndexBuilds(TOperationClient& client, const TString& dbPath) { - const ui64 pageSize = 100; - TString pageToken; - - do { - const ui32 maxRetries = 8; - TDuration retrySleep = TDuration::MilliSeconds(100); - - for (ui32 retryNumber = 0; retryNumber <= maxRetries; ++retryNumber) { - auto operations = client.List<TBuildIndexOperation>(pageSize, pageToken).GetValueSync(); - - if (!operations.IsSuccess()) { - if (retryNumber == maxRetries) { - Y_ENSURE(false, "Cannot list operations"); - } - - switch (operations.GetStatus()) { - case EStatus::ABORTED: - break; - case EStatus::OVERLOADED: - case EStatus::CLIENT_RESOURCE_EXHAUSTED: - case EStatus::UNAVAILABLE: - case EStatus::TRANSPORT_UNAVAILABLE: - ExponentialBackoff(retrySleep); - break; - default: - Y_ENSURE(false, "Unexpected status while trying to list operations: " << operations.GetStatus()); - } - - continue; - } - - for (const auto& operation : operations.GetList()) { - if (operation.Metadata().Path != dbPath) { - continue; - } - - switch (operation.Metadata().State) { - case EBuildIndexState::Preparing: - case EBuildIndexState::TransferData: - case EBuildIndexState::Applying: - case EBuildIndexState::Cancellation: - return true; - default: - break; - } - } - - pageToken = operations.NextPageToken(); - } - } while (pageToken != "0"); - - return false; -} - -} // anonymous - -TRestoreClient::TRestoreClient( - TImportClient& importClient, - TOperationClient& operationClient, +using namespace NTable; + +extern const char DOC_API_TABLE_VERSION_ATTR[] = "__document_api_version"; +extern const char DOC_API_REQUEST_TYPE[] = "_document_api_request"; + +namespace { + +bool IsFileExists(const TFsPath& path) { + return path.Exists() && path.IsFile(); +} + +Ydb::Table::CreateTableRequest ReadTableScheme(const TString& fsPath) { + Ydb::Table::CreateTableRequest proto; + Y_ENSURE(google::protobuf::TextFormat::ParseFromString(TFileInput(fsPath).ReadAll(), &proto)); + return proto; +} + +TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) { + return TProtoAccessor::FromProto(proto); +} + +bool HasRunningIndexBuilds(TOperationClient& client, const TString& dbPath) { + const ui64 pageSize = 100; + TString pageToken; + + do { + const ui32 maxRetries = 8; + TDuration retrySleep = TDuration::MilliSeconds(100); + + for (ui32 retryNumber = 0; retryNumber <= maxRetries; ++retryNumber) { + auto operations = client.List<TBuildIndexOperation>(pageSize, pageToken).GetValueSync(); + + if (!operations.IsSuccess()) { + if (retryNumber == maxRetries) { + Y_ENSURE(false, "Cannot list operations"); + } + + switch (operations.GetStatus()) { + case EStatus::ABORTED: + break; + case EStatus::OVERLOADED: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + case EStatus::UNAVAILABLE: + case EStatus::TRANSPORT_UNAVAILABLE: + ExponentialBackoff(retrySleep); + break; + default: + Y_ENSURE(false, "Unexpected status while trying to list operations: " << operations.GetStatus()); + } + + continue; + } + + for (const auto& operation : operations.GetList()) { + if (operation.Metadata().Path != dbPath) { + continue; + } + + switch (operation.Metadata().State) { + case EBuildIndexState::Preparing: + case EBuildIndexState::TransferData: + case EBuildIndexState::Applying: + case EBuildIndexState::Cancellation: + return true; + default: + break; + } + } + + pageToken = operations.NextPageToken(); + } + } while (pageToken != "0"); + + return false; +} + +} // anonymous + +TRestoreClient::TRestoreClient( + TImportClient& importClient, + TOperationClient& operationClient, TSchemeClient& schemeClient, - TTableClient& tableClient) - : ImportClient(importClient) - , OperationClient(operationClient) + TTableClient& tableClient) + : ImportClient(importClient) + , OperationClient(operationClient) , SchemeClient(schemeClient) - , TableClient(tableClient) -{ -} - -TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { - // find existing items + , TableClient(tableClient) +{ +} + +TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) { + // find existing items TFsPath dbBasePath = dbPath; while (true) { - auto result = DescribePath(SchemeClient, dbBasePath).GetStatus(); + auto result = DescribePath(SchemeClient, dbBasePath).GetStatus(); if (result == EStatus::SUCCESS) { break; } if (result != EStatus::SCHEME_ERROR) { - return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not find existing path"); + return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not find existing path"); } dbBasePath = dbBasePath.Parent(); } - - auto oldDirectoryList = SchemeClient.ListDirectory(dbBasePath).GetValueSync(); - if (!oldDirectoryList.IsSuccess()) { - return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not list existing directory"); + + auto oldDirectoryList = SchemeClient.ListDirectory(dbBasePath).GetValueSync(); + if (!oldDirectoryList.IsSuccess()) { + return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not list existing directory"); } THashSet<TString> oldEntries; - for (const auto& entry : oldDirectoryList.GetChildren()) { + for (const auto& entry : oldDirectoryList.GetChildren()) { oldEntries.insert(entry.Name); } - // restore + // restore auto restoreResult = RestoreFolder(fsPath, dbPath, settings); if (restoreResult.IsSuccess() || settings.SavePartialResult_) { return restoreResult; } - // cleanup + // cleanup auto newDirectoryList = SchemeClient.ListDirectory(dbBasePath).GetValueSync(); if (!newDirectoryList.IsSuccess()) { return restoreResult; } for (const auto& entry : newDirectoryList.GetChildren()) { - if (oldEntries.contains(entry.Name)) { - continue; - } - - auto fullPath = dbBasePath.Child(entry.Name); - - switch (entry.Type) { - case ESchemeEntryType::Directory: { + if (oldEntries.contains(entry.Name)) { + continue; + } + + auto fullPath = dbBasePath.Child(entry.Name); + + switch (entry.Type) { + case ESchemeEntryType::Directory: { auto result = RemoveDirectoryRecursive(TableClient, SchemeClient, fullPath, {}, true); - if (!result.IsSuccess()) { - return restoreResult; + if (!result.IsSuccess()) { + return restoreResult; } - break; - } - case ESchemeEntryType::Table: { + break; + } + case ESchemeEntryType::Table: { auto result = TableClient.RetryOperationSync([path = fullPath](TSession session) { - return session.DropTable(path).GetValueSync(); - }); - if (!result.IsSuccess()) { - return restoreResult; + return session.DropTable(path).GetValueSync(); + }); + if (!result.IsSuccess()) { + return restoreResult; } - break; + break; } - default: - return restoreResult; + default: + return restoreResult; } } return restoreResult; -} - -TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) { - if (!fsPath) { - return Result<TRestoreResult>(EStatus::BAD_REQUEST, "Folder is not specified"); - } - - if (!fsPath.Exists()) { - return Result<TRestoreResult>(EStatus::BAD_REQUEST, - TStringBuilder() << "Specified folder does not exist: " << fsPath.GetPath()); - } - - if (!fsPath.IsDirectory()) { - return Result<TRestoreResult>(EStatus::BAD_REQUEST, - TStringBuilder() << "Specified folder is not a directory: " << fsPath.GetPath()); - } - - if (IsFileExists(fsPath.Child(INCOMPLETE_FILE_NAME))) { - return Result<TRestoreResult>(EStatus::BAD_REQUEST, - TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath()); - } - - if (IsFileExists(fsPath.Child(SCHEME_FILE_NAME))) { - return RestoreTable(fsPath, Join('/', dbPath, fsPath.GetName()), settings); - } - - if (IsFileExists(fsPath.Child(EMPTY_FILE_NAME))) { - return MakeDirectory(SchemeClient, Join('/', dbPath, fsPath.GetName())); - } - - TMaybe<TRestoreResult> result; - - TVector<TFsPath> children; - fsPath.List(children); - for (const auto& child : children) { - if (IsFileExists(child.Child(SCHEME_FILE_NAME))) { - result = RestoreTable(child, Join('/', dbPath, child.GetName()), settings); - } else if (IsFileExists(child.Child(EMPTY_FILE_NAME))) { - result = MakeDirectory(SchemeClient, Join('/', dbPath, child.GetName())); - } else { - result = RestoreFolder(child, Join('/', dbPath, child.GetName()), settings); - } - - if (!result->IsSuccess()) { - return *result; - } - } - - if (!result) { - return Result<TRestoreResult>(EStatus::SUCCESS); - } - - return *result; -} - -TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) { - if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) { - return Result<TRestoreResult>(EStatus::BAD_REQUEST, - TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath()); - } - - auto scheme = ReadTableScheme(fsPath.Child(SCHEME_FILE_NAME)); - auto dumpedDesc = TableDescriptionFromProto(scheme); - - if (settings.DryRun_) { - return CheckSchema(dbPath, dumpedDesc); - } - - if (settings.RestoreData_) { - auto woIndexes = scheme; - woIndexes.clear_indexes(); - - auto result = RestoreData(fsPath, dbPath, settings, TableDescriptionFromProto(woIndexes)); - if (!result.IsSuccess()) { - return result; - } - } - - if (settings.RestoreIndexes_) { - auto result = RestoreIndexes(dbPath, dumpedDesc); - if (!result.IsSuccess()) { - return result; - } - } - - return Result<TRestoreResult>(); -} - -TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDescription& desc) { - TMaybe<TTableDescription> actualDesc; - auto descResult = DescribeTable(TableClient, dbPath, actualDesc); - if (!descResult.IsSuccess()) { - return Result<TRestoreResult>(std::move(descResult)); - } - - auto unorderedColumns = [](const TVector<TColumn>& orderedColumns) { - THashMap<TString, TColumn> result; - for (const auto& column : orderedColumns) { - result.emplace(column.Name, column); - } - return result; - }; - - auto unorderedIndexes = [](const TVector<TIndexDescription>& orderedIndexes) { - THashMap<TString, TIndexDescription> result; - for (const auto& index : orderedIndexes) { - result.emplace(index.GetIndexName(), index); - } - return result; - }; - - if (unorderedColumns(desc.GetColumns()) != unorderedColumns(actualDesc->GetColumns())) { - return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Columns differ" - << ": dumped# " << JoinSeq(", ", desc.GetColumns()) - << ", actual# " << JoinSeq(", ", actualDesc->GetColumns())); - } - - if (desc.GetPrimaryKeyColumns() != actualDesc->GetPrimaryKeyColumns()) { - return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Primary key columns differ" - << ": dumped# " << JoinSeq(", ", desc.GetPrimaryKeyColumns()) - << ", actual# " << JoinSeq(", ", actualDesc->GetPrimaryKeyColumns())); - } - - if (unorderedIndexes(desc.GetIndexDescriptions()) != unorderedIndexes(actualDesc->GetIndexDescriptions())) { - return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Indexes differ" - << ": dumped# " << JoinSeq(", ", desc.GetIndexDescriptions()) - << ", actual# " << JoinSeq(", ", actualDesc->GetIndexDescriptions())); - } - - return Result<TRestoreResult>(); -} - -TRestoreResult TRestoreClient::RestoreData(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const TTableDescription& desc) { - if (desc.GetAttributes().contains(DOC_API_TABLE_VERSION_ATTR) && settings.SkipDocumentTables_) { - return Result<TRestoreResult>(); - } - - auto createResult = TableClient.RetryOperationSync([&dbPath, &desc](TSession session) { - return session.CreateTable(dbPath, TTableDescription(desc), - TCreateTableSettings().RequestType(DOC_API_REQUEST_TYPE)).GetValueSync(); - }); - if (!createResult.IsSuccess()) { - return Result<TRestoreResult>(std::move(createResult)); - } - - THolder<NPrivate::IDataAccumulator> accumulator; - THolder<NPrivate::IDataWriter> writer; - - switch (settings.Mode_) { - case TRestoreSettings::EMode::Yql: - case TRestoreSettings::EMode::BulkUpsert: { +} + +TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) { + if (!fsPath) { + return Result<TRestoreResult>(EStatus::BAD_REQUEST, "Folder is not specified"); + } + + if (!fsPath.Exists()) { + return Result<TRestoreResult>(EStatus::BAD_REQUEST, + TStringBuilder() << "Specified folder does not exist: " << fsPath.GetPath()); + } + + if (!fsPath.IsDirectory()) { + return Result<TRestoreResult>(EStatus::BAD_REQUEST, + TStringBuilder() << "Specified folder is not a directory: " << fsPath.GetPath()); + } + + if (IsFileExists(fsPath.Child(INCOMPLETE_FILE_NAME))) { + return Result<TRestoreResult>(EStatus::BAD_REQUEST, + TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath()); + } + + if (IsFileExists(fsPath.Child(SCHEME_FILE_NAME))) { + return RestoreTable(fsPath, Join('/', dbPath, fsPath.GetName()), settings); + } + + if (IsFileExists(fsPath.Child(EMPTY_FILE_NAME))) { + return MakeDirectory(SchemeClient, Join('/', dbPath, fsPath.GetName())); + } + + TMaybe<TRestoreResult> result; + + TVector<TFsPath> children; + fsPath.List(children); + for (const auto& child : children) { + if (IsFileExists(child.Child(SCHEME_FILE_NAME))) { + result = RestoreTable(child, Join('/', dbPath, child.GetName()), settings); + } else if (IsFileExists(child.Child(EMPTY_FILE_NAME))) { + result = MakeDirectory(SchemeClient, Join('/', dbPath, child.GetName())); + } else { + result = RestoreFolder(child, Join('/', dbPath, child.GetName()), settings); + } + + if (!result->IsSuccess()) { + return *result; + } + } + + if (!result) { + return Result<TRestoreResult>(EStatus::SUCCESS); + } + + return *result; +} + +TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) { + if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) { + return Result<TRestoreResult>(EStatus::BAD_REQUEST, + TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath()); + } + + auto scheme = ReadTableScheme(fsPath.Child(SCHEME_FILE_NAME)); + auto dumpedDesc = TableDescriptionFromProto(scheme); + + if (settings.DryRun_) { + return CheckSchema(dbPath, dumpedDesc); + } + + if (settings.RestoreData_) { + auto woIndexes = scheme; + woIndexes.clear_indexes(); + + auto result = RestoreData(fsPath, dbPath, settings, TableDescriptionFromProto(woIndexes)); + if (!result.IsSuccess()) { + return result; + } + } + + if (settings.RestoreIndexes_) { + auto result = RestoreIndexes(dbPath, dumpedDesc); + if (!result.IsSuccess()) { + return result; + } + } + + return Result<TRestoreResult>(); +} + +TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDescription& desc) { + TMaybe<TTableDescription> actualDesc; + auto descResult = DescribeTable(TableClient, dbPath, actualDesc); + if (!descResult.IsSuccess()) { + return Result<TRestoreResult>(std::move(descResult)); + } + + auto unorderedColumns = [](const TVector<TColumn>& orderedColumns) { + THashMap<TString, TColumn> result; + for (const auto& column : orderedColumns) { + result.emplace(column.Name, column); + } + return result; + }; + + auto unorderedIndexes = [](const TVector<TIndexDescription>& orderedIndexes) { + THashMap<TString, TIndexDescription> result; + for (const auto& index : orderedIndexes) { + result.emplace(index.GetIndexName(), index); + } + return result; + }; + + if (unorderedColumns(desc.GetColumns()) != unorderedColumns(actualDesc->GetColumns())) { + return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Columns differ" + << ": dumped# " << JoinSeq(", ", desc.GetColumns()) + << ", actual# " << JoinSeq(", ", actualDesc->GetColumns())); + } + + if (desc.GetPrimaryKeyColumns() != actualDesc->GetPrimaryKeyColumns()) { + return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Primary key columns differ" + << ": dumped# " << JoinSeq(", ", desc.GetPrimaryKeyColumns()) + << ", actual# " << JoinSeq(", ", actualDesc->GetPrimaryKeyColumns())); + } + + if (unorderedIndexes(desc.GetIndexDescriptions()) != unorderedIndexes(actualDesc->GetIndexDescriptions())) { + return Result<TRestoreResult>(EStatus::SCHEME_ERROR, TStringBuilder() << "Indexes differ" + << ": dumped# " << JoinSeq(", ", desc.GetIndexDescriptions()) + << ", actual# " << JoinSeq(", ", actualDesc->GetIndexDescriptions())); + } + + return Result<TRestoreResult>(); +} + +TRestoreResult TRestoreClient::RestoreData(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const TTableDescription& desc) { + if (desc.GetAttributes().contains(DOC_API_TABLE_VERSION_ATTR) && settings.SkipDocumentTables_) { + return Result<TRestoreResult>(); + } + + auto createResult = TableClient.RetryOperationSync([&dbPath, &desc](TSession session) { + return session.CreateTable(dbPath, TTableDescription(desc), + TCreateTableSettings().RequestType(DOC_API_REQUEST_TYPE)).GetValueSync(); + }); + if (!createResult.IsSuccess()) { + return Result<TRestoreResult>(std::move(createResult)); + } + + THolder<NPrivate::IDataAccumulator> accumulator; + THolder<NPrivate::IDataWriter> writer; + + switch (settings.Mode_) { + case TRestoreSettings::EMode::Yql: + case TRestoreSettings::EMode::BulkUpsert: { accumulator.Reset(CreateCompatAccumulator(dbPath, desc, settings)); writer.Reset(CreateCompatWriter(dbPath, TableClient, accumulator.Get(), settings)); - - break; - } - - case TRestoreSettings::EMode::ImportData: { - TMaybe<TTableDescription> actualDesc; - auto descResult = DescribeTable(TableClient, dbPath, actualDesc); - if (!descResult.IsSuccess()) { - return Result<TRestoreResult>(std::move(descResult)); - } - + + break; + } + + case TRestoreSettings::EMode::ImportData: { + TMaybe<TTableDescription> actualDesc; + auto descResult = DescribeTable(TableClient, dbPath, actualDesc); + if (!descResult.IsSuccess()) { + return Result<TRestoreResult>(std::move(descResult)); + } + accumulator.Reset(CreateImportDataAccumulator(desc, *actualDesc, settings)); writer.Reset(CreateImportDataWriter(dbPath, desc, ImportClient, TableClient, accumulator.Get(), settings)); - - break; - } - } - - ui32 dataFileId = 0; - TFsPath dataFile = fsPath.Child(DataFileName(dataFileId)); - - while (dataFile.Exists()) { - TFileInput input(dataFile, settings.FileBufferSize_); - TString line; - - while (input.ReadLine(line)) { - while (!accumulator->Fits(line)) { - if (!accumulator->Ready(true)) { - return Result<TRestoreResult>(EStatus::INTERNAL_ERROR, "Data is not ready"); - } - - if (!writer->Push(accumulator->GetData(true))) { - return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #1"); - } - } - - accumulator->Feed(std::move(line)); - if (accumulator->Ready()) { - if (!writer->Push(accumulator->GetData())) { - return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #2"); - } - } - } - - dataFile = fsPath.Child(DataFileName(++dataFileId)); - } - - while (accumulator->Ready(true)) { - if (!writer->Push(accumulator->GetData(true))) { - return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #3"); - } - } - - writer->Wait(); - - return Result<TRestoreResult>(); -} - -TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTableDescription& desc) { - TMaybe<TTableDescription> actualDesc; - - for (const auto& index : desc.GetIndexDescriptions()) { - // check (and wait) for unuexpected index buils - while (HasRunningIndexBuilds(OperationClient, dbPath)) { - actualDesc.Clear(); - Sleep(TDuration::Minutes(1)); - } - - if (!actualDesc) { - auto descResult = DescribeTable(TableClient, dbPath, actualDesc); - if (!descResult.IsSuccess()) { - return Result<TRestoreResult>(std::move(descResult)); - } - } - - if (FindPtr(actualDesc->GetIndexDescriptions(), index)) { - continue; - } - - auto status = TableClient.RetryOperationSync([&dbPath, &index](TSession session) { - auto settings = TAlterTableSettings().AppendAddIndexes(index); - return session.AlterTableLong(dbPath, settings).GetValueSync().Status(); - }); - if (!status.IsSuccess() && status.GetStatus() != EStatus::STATUS_UNDEFINED) { - return Result<TRestoreResult>(std::move(status)); - } - - // wait for expected index build - while (HasRunningIndexBuilds(OperationClient, dbPath)) { - Sleep(TDuration::Minutes(1)); - } - } - - return Result<TRestoreResult>(); -} - -} // NDump -} // NYdb + + break; + } + } + + ui32 dataFileId = 0; + TFsPath dataFile = fsPath.Child(DataFileName(dataFileId)); + + while (dataFile.Exists()) { + TFileInput input(dataFile, settings.FileBufferSize_); + TString line; + + while (input.ReadLine(line)) { + while (!accumulator->Fits(line)) { + if (!accumulator->Ready(true)) { + return Result<TRestoreResult>(EStatus::INTERNAL_ERROR, "Data is not ready"); + } + + if (!writer->Push(accumulator->GetData(true))) { + return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #1"); + } + } + + accumulator->Feed(std::move(line)); + if (accumulator->Ready()) { + if (!writer->Push(accumulator->GetData())) { + return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #2"); + } + } + } + + dataFile = fsPath.Child(DataFileName(++dataFileId)); + } + + while (accumulator->Ready(true)) { + if (!writer->Push(accumulator->GetData(true))) { + return Result<TRestoreResult>(EStatus::GENERIC_ERROR, "Cannot write data #3"); + } + } + + writer->Wait(); + + return Result<TRestoreResult>(); +} + +TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTableDescription& desc) { + TMaybe<TTableDescription> actualDesc; + + for (const auto& index : desc.GetIndexDescriptions()) { + // check (and wait) for unuexpected index buils + while (HasRunningIndexBuilds(OperationClient, dbPath)) { + actualDesc.Clear(); + Sleep(TDuration::Minutes(1)); + } + + if (!actualDesc) { + auto descResult = DescribeTable(TableClient, dbPath, actualDesc); + if (!descResult.IsSuccess()) { + return Result<TRestoreResult>(std::move(descResult)); + } + } + + if (FindPtr(actualDesc->GetIndexDescriptions(), index)) { + continue; + } + + auto status = TableClient.RetryOperationSync([&dbPath, &index](TSession session) { + auto settings = TAlterTableSettings().AppendAddIndexes(index); + return session.AlterTableLong(dbPath, settings).GetValueSync().Status(); + }); + if (!status.IsSuccess() && status.GetStatus() != EStatus::STATUS_UNDEFINED) { + return Result<TRestoreResult>(std::move(status)); + } + + // wait for expected index build + while (HasRunningIndexBuilds(OperationClient, dbPath)) { + Sleep(TDuration::Minutes(1)); + } + } + + return Result<TRestoreResult>(); +} + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.h b/ydb/public/lib/ydb_cli/dump/restore_impl.h index db59fc0ea1c..a0109371706 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_impl.h +++ b/ydb/public/lib/ydb_cli/dump/restore_impl.h @@ -1,64 +1,64 @@ -#pragma once - -#include "dump.h" - +#pragma once + +#include "dump.h" + #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <util/folder/path.h> - -namespace NYdb { -namespace NDump { - -extern const char DOC_API_TABLE_VERSION_ATTR[23]; -extern const char DOC_API_REQUEST_TYPE[22]; - -namespace NPrivate { - -class IDataAccumulator { -public: - virtual ~IDataAccumulator() = default; - virtual bool Fits(const TString& line) const = 0; - virtual void Feed(TString&& line) = 0; - virtual bool Ready(bool force = false) const = 0; - virtual TString GetData(bool force = false) = 0; -}; - -class IDataWriter { -public: - virtual ~IDataWriter() = default; - virtual bool Push(TString&& data) = 0; - virtual void Wait() = 0; -}; - -} // NPrivate - -class TRestoreClient { - TRestoreResult RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings); - TRestoreResult RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings); - - TRestoreResult CheckSchema(const TString& dbPath, const NTable::TTableDescription& desc); - TRestoreResult RestoreData(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const NTable::TTableDescription& desc); - TRestoreResult RestoreIndexes(const TString& dbPath, const NTable::TTableDescription& desc); - -public: - explicit TRestoreClient( - NImport::TImportClient& importClient, - NOperation::TOperationClient& operationClient, + +#include <util/folder/path.h> + +namespace NYdb { +namespace NDump { + +extern const char DOC_API_TABLE_VERSION_ATTR[23]; +extern const char DOC_API_REQUEST_TYPE[22]; + +namespace NPrivate { + +class IDataAccumulator { +public: + virtual ~IDataAccumulator() = default; + virtual bool Fits(const TString& line) const = 0; + virtual void Feed(TString&& line) = 0; + virtual bool Ready(bool force = false) const = 0; + virtual TString GetData(bool force = false) = 0; +}; + +class IDataWriter { +public: + virtual ~IDataWriter() = default; + virtual bool Push(TString&& data) = 0; + virtual void Wait() = 0; +}; + +} // NPrivate + +class TRestoreClient { + TRestoreResult RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings); + TRestoreResult RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings); + + TRestoreResult CheckSchema(const TString& dbPath, const NTable::TTableDescription& desc); + TRestoreResult RestoreData(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const NTable::TTableDescription& desc); + TRestoreResult RestoreIndexes(const TString& dbPath, const NTable::TTableDescription& desc); + +public: + explicit TRestoreClient( + NImport::TImportClient& importClient, + NOperation::TOperationClient& operationClient, NScheme::TSchemeClient& SchemeClient, - NTable::TTableClient& tableClient); - - TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {}); - -private: - NImport::TImportClient& ImportClient; - NOperation::TOperationClient& OperationClient; + NTable::TTableClient& tableClient); + + TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {}); + +private: + NImport::TImportClient& ImportClient; + NOperation::TOperationClient& OperationClient; NScheme::TSchemeClient& SchemeClient; - NTable::TTableClient& TableClient; - -}; // TRestoreClient - -} // NDump -} // NYdb + NTable::TTableClient& TableClient; + +}; // TRestoreClient + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp b/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp index 5138ab2895f..817aa76eea3 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp +++ b/ydb/public/lib/ydb_cli/dump/restore_import_data.cpp @@ -1,910 +1,910 @@ -#include "restore_import_data.h" - +#include "restore_import_data.h" + #include <ydb/public/lib/ydb_cli/dump/util/util.h> -#include <library/cpp/string_utils/quote/quote.h> +#include <library/cpp/string_utils/quote/quote.h> #include <library/cpp/bucket_quoter/bucket_quoter.h> - -#include <util/datetime/base.h> -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <util/generic/variant.h> -#include <util/generic/vector.h> -#include <util/stream/str.h> -#include <util/string/builder.h> -#include <util/string/cast.h> -#include <util/system/mutex.h> -#include <util/thread/pool.h> - -namespace NYdb { -namespace NDump { - -using namespace NImport; -using namespace NTable; - -namespace { - -class TValue { - struct TSpecialType { - bool operator<(const TSpecialType&) const { + +#include <util/datetime/base.h> +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <util/generic/variant.h> +#include <util/generic/vector.h> +#include <util/stream/str.h> +#include <util/string/builder.h> +#include <util/string/cast.h> +#include <util/system/mutex.h> +#include <util/thread/pool.h> + +namespace NYdb { +namespace NDump { + +using namespace NImport; +using namespace NTable; + +namespace { + +class TValue { + struct TSpecialType { + bool operator<(const TSpecialType&) const { return false; // to make std::variant's comparator happy - } - }; - - struct TInf: public TSpecialType {}; // we need just the positive one, it'll be used as a last boundary - struct TNull: public TSpecialType {}; - - enum class EType { - Inf = 0, - Null, - String, - Pod, - }; - - inline EType GetType() const { - switch (Value.index()) { - case 0: - return EType::Inf; - case 1: - return EType::Null; - case 2: - return EType::String; - default: - return EType::Pod; - } - } - -public: - template <typename T> - explicit TValue(T value) - : Value(value) - { - } - - static TValue Inf() { - return TValue(TInf()); - } - - static TValue Null() { - return TValue(TNull()); - } - - bool operator<(const TValue& rhs) const { - const EType lType = GetType(); - const EType rType = rhs.GetType(); - - if (lType == EType::Inf) { - return false; - } else if (rType == EType::Inf) { - return true; - } else if (lType == EType::Null) { - return lType != rType; - } else if (rType == EType::Null) { - return false; - } else { - return Value < rhs.Value; - } - } - - ui64 MemSize() const { - switch (GetType()) { - case EType::String: + } + }; + + struct TInf: public TSpecialType {}; // we need just the positive one, it'll be used as a last boundary + struct TNull: public TSpecialType {}; + + enum class EType { + Inf = 0, + Null, + String, + Pod, + }; + + inline EType GetType() const { + switch (Value.index()) { + case 0: + return EType::Inf; + case 1: + return EType::Null; + case 2: + return EType::String; + default: + return EType::Pod; + } + } + +public: + template <typename T> + explicit TValue(T value) + : Value(value) + { + } + + static TValue Inf() { + return TValue(TInf()); + } + + static TValue Null() { + return TValue(TNull()); + } + + bool operator<(const TValue& rhs) const { + const EType lType = GetType(); + const EType rType = rhs.GetType(); + + if (lType == EType::Inf) { + return false; + } else if (rType == EType::Inf) { + return true; + } else if (lType == EType::Null) { + return lType != rType; + } else if (rType == EType::Null) { + return false; + } else { + return Value < rhs.Value; + } + } + + ui64 MemSize() const { + switch (GetType()) { + case EType::String: return sizeof(Value) + std::get<TString>(Value).size(); - default: - return sizeof(Value); - } - } - -private: + default: + return sizeof(Value); + } + } + +private: std::variant< - TInf, - TNull, - TString, - bool, - ui8, - i32, - ui32, - i64, - ui64 - > Value; - -}; // TValue - -template <typename TParser> -class TValueConverter { - TValue Convert(EPrimitiveType type) const { - switch (type) { - case EPrimitiveType::Bool: - return TValue(Parser.GetBool()); - case EPrimitiveType::Uint8: - return TValue(Parser.GetUint8()); - case EPrimitiveType::Int32: - return TValue(Parser.GetInt32()); - case EPrimitiveType::Uint32: - return TValue(Parser.GetUint32()); - case EPrimitiveType::Int64: - return TValue(Parser.GetInt64()); - case EPrimitiveType::Uint64: - return TValue(Parser.GetUint64()); - case EPrimitiveType::DyNumber: - return TValue(Parser.GetDyNumber()); - case EPrimitiveType::Date: - return TValue(Parser.GetDate().GetValue()); - case EPrimitiveType::Datetime: - return TValue(Parser.GetDatetime().GetValue()); - case EPrimitiveType::Timestamp: - return TValue(Parser.GetTimestamp().GetValue()); - case EPrimitiveType::String: - return TValue(Parser.GetString()); - case EPrimitiveType::Utf8: - return TValue(Parser.GetUtf8()); - default: - Y_ENSURE(false, "Unexpected primitive type: " << type); - } - } - - void Convert(TVector<TValue>& value) { - switch (Parser.GetKind()) { - case TTypeParser::ETypeKind::Primitive: - value.emplace_back(Convert(Parser.GetPrimitiveType())); - break; - - case TTypeParser::ETypeKind::Optional: - Parser.OpenOptional(); - if (Parser.IsNull()) { - value.emplace_back(TValue::Null()); - } else { - Convert(value); - } - Parser.CloseOptional(); - break; - - case TTypeParser::ETypeKind::Tuple: - Parser.OpenTuple(); - while (Parser.TryNextElement()) { - Convert(value); - } - Parser.CloseTuple(); - break; - - default: - Y_ENSURE(false, "Unexpected type kind: " << Parser.GetKind()); - } - } - -public: - explicit TValueConverter(TParser& parser) - : Parser(parser) - { - } - - TVector<TValue> ConvertAll() { - TVector<TValue> value; - Convert(value); - return value; - } - - TValue ConvertSingle() { - auto value = ConvertAll(); - Y_ENSURE(value.size() == 1, "Unexpected value size: " << value.size()); - return value[0]; - } - -private: - TParser& Parser; - -}; // TValueConverter - -class TYdbDumpValueParser { - TString CheckedUnescape() const { - Y_ENSURE(Value.size() >= 2 || Value.front() == '"' || Value.back() == '"'); - return CGIUnescapeRet(TStringBuf(Value).Skip(1).Chop(1)); - } - -public: - explicit TYdbDumpValueParser(TStringBuf value, EPrimitiveType type) - : Value(value) - , Type(type) - , OptionalOpened(false) - { - } - - TTypeParser::ETypeKind GetKind() const { - return OptionalOpened - ? TTypeParser::ETypeKind::Primitive - : TTypeParser::ETypeKind::Optional; - } - - EPrimitiveType GetPrimitiveType() const { - return Type; - } - - bool GetBool() const { - return FromString<bool>(Value); - } - - ui8 GetUint8() const { - return FromString<ui8>(Value); - } - - i32 GetInt32() const { - return FromString<i32>(Value); - } - - ui32 GetUint32() const { - return FromString<ui32>(Value); - } - - i64 GetInt64() const { - return FromString<i64>(Value); - } - - ui64 GetUint64() const { - return FromString<ui64>(Value); - } - - TString GetDyNumber() const { - return ToString(Value); - } - - TInstant GetDate() const { - return TInstant::ParseIso8601(Value); - } - - TInstant GetDatetime() const { - return TInstant::ParseIso8601(Value); - } - - TInstant GetTimestamp() const { - return TInstant::ParseIso8601(Value); - } - - TString GetString() const { - return CheckedUnescape(); - } - - TString GetUtf8() const { - return CheckedUnescape(); - } - - bool IsNull() const { - return Value == "null"; - } - - void OpenOptional() { - OptionalOpened = true; - } - - void CloseOptional() { - OptionalOpened = false; - } - - bool TryNextElement() { - return false; - } - - void OpenTuple() {} - void CloseTuple() {} - -private: - const TStringBuf Value; - const EPrimitiveType Type; - - bool OptionalOpened; - -}; // TYdbDumpValueParser - -class TKey: public TVector<TValue> { -public: - using TVector<TValue>::TVector; - - TKey(TVector<TValue>&& values) - : TVector<TValue>(std::move(values)) - { - } - - bool operator<(const TKey& rhs) const { - for (ui32 i = 0; i < Min(size(), rhs.size()); ++i) { - if (at(i) < rhs.at(i)) { - return true; - } - - if (rhs.at(i) < at(i)) { - return false; - } - } - - return size() < rhs.size(); - } - - ui64 MemSize() const { - return Accumulate(begin(), end(), ui64(0), [](ui64 s, const auto& v) { - return s + v.MemSize(); - }); - } - -}; // TKey - -using TSplitPoint = TKey; - -class TKeyBuilder { - static auto MakeKeyColumnIds(const TVector<TString>& keyColumns) { - THashMap<TString, ui32> keyColumnIds; - - for (ui32 i = 0; i < keyColumns.size(); ++i) { - Y_ENSURE(keyColumnIds.emplace(keyColumns.at(i), i).second); - } - - return keyColumnIds; - } - - static EPrimitiveType GetPrimitiveType(TTypeParser& parser) { - switch (parser.GetKind()) { - case TTypeParser::ETypeKind::Optional: - parser.OpenOptional(); - return GetPrimitiveType(parser); - case TTypeParser::ETypeKind::Primitive: - return parser.GetPrimitive(); - default: - Y_ENSURE(false, "Unexpected type kind: " << parser.GetKind()); - } - } - - static EPrimitiveType GetPrimitiveType(const TType& type) { - TTypeParser parser(type); - return GetPrimitiveType(parser); - } - -public: - explicit TKeyBuilder(const TVector<TColumn>& columns, const TVector<TString>& keyColumns) - : Columns(columns) - , KeyColumnIds(MakeKeyColumnIds(keyColumns)) - { - } - - TKey Build(TStringBuf line) const { - TMap<ui32, TValue> values; - - const TStringBuf origLine = line; - for (const auto& column : Columns) { - TStringBuf value = line.NextTok(','); - Y_ENSURE(value, "Empty token on line: " << origLine); - - auto it = KeyColumnIds.find(column.Name); - if (it == KeyColumnIds.end()) { - continue; - } - - TYdbDumpValueParser parser(value, GetPrimitiveType(column.Type)); - values.emplace(it->second, TValueConverter<TYdbDumpValueParser>(parser).ConvertSingle()); - } - - TKey key; - for (auto& [_, value] : values) { - key.push_back(std::move(value)); - } - - return key; - } - -private: - const TVector<TColumn> Columns; - const THashMap<TString, ui32> KeyColumnIds; - -}; // TKeyBuilder - -ui64 AdjustedRecordSize(ui64 recordSize) { - return recordSize + 1 /* \n */; -} - -class TPartitionRows { - void IncSize(ui64 keySize, ui64 recordSize) { - MemSizeCounter += (keySize + recordSize); - RecordsSizeCounter += AdjustedRecordSize(recordSize); - } - - void DecSize(ui64 keySize, ui64 recordSize) { - MemSizeCounter -= (keySize + recordSize); - RecordsSizeCounter -= AdjustedRecordSize(recordSize); - } - -public: - TPartitionRows() - : MemSizeCounter(0) - , RecordsSizeCounter(0) - { - } - - ui64 MemSize() const { - return MemSizeCounter; - } - - ui64 RecordsSize() const { - return RecordsSizeCounter; - } - - bool Add(TKey&& key, TString&& record) { - auto ret = Rows.emplace(std::move(key), std::move(record)); - if (!ret.second) { - return ret.second; - } - - IncSize(ret.first->first.MemSize(), ret.first->second.size()); - return ret.second; - } - - bool Empty() const { - return Rows.empty(); - } - - auto Pop() { - Y_ENSURE(!Empty()); - auto ret = Rows.extract(Rows.begin()); - - DecSize(ret.key().MemSize(), ret.mapped().size()); - return ret; - } - - TString Serialize(ui64 maxSize) { - TStringBuilder result; - - if (Empty()) { - return result; - } - - auto handle = Pop(); - while (result.size() + handle.mapped().size() < maxSize) { - result << handle.mapped() << "\n"; - - if (Empty()) { - return result; - } - - handle = Pop(); - } - - Add(std::move(handle.key()), std::move(handle.mapped())); - return result; - } - -private: - TMap<TKey, TString> Rows; - ui64 MemSizeCounter; - ui64 RecordsSizeCounter; - -}; // TPartitionRows - -class TTableRows { - template <typename TIterator> - struct TIteratorHash { - size_t operator()(const TIterator& x) const { - return THash<intptr_t>()(reinterpret_cast<intptr_t>(&*x)); - } - }; - - // key is a right-side boundary of partition - using TRows = TMap<TSplitPoint, TPartitionRows>; - using TRowsBy = TMap<ui64, THashSet<TRows::iterator, TIteratorHash<TRows::iterator>>, TGreater<ui64>>; - - static auto MakeSplitPoints(const TVector<TKeyRange>& keyRanges) { - Y_ENSURE(!keyRanges.empty()); - - TVector<TSplitPoint> splitPoints; - - auto it = keyRanges.begin(); - while (++it != keyRanges.end()) { - const auto& from = it->From(); - - Y_ENSURE(from.Defined()); - Y_ENSURE(from->IsInclusive()); - - TValueParser parser(from->GetValue()); - splitPoints.push_back(TValueConverter<TValueParser>(parser).ConvertAll()); - } - - return splitPoints; - } - - static auto MakeEmptyRows(TVector<TSplitPoint>&& splitPoints) { - TRows rows; - - for (auto& splitPoint : splitPoints) { - rows.emplace(std::move(splitPoint), TPartitionRows()); - } - - // last boundary is +inf - rows.emplace(TSplitPoint{TValue::Inf()}, TPartitionRows()); - - return rows; - } - - template <typename T> - static auto FindPartition(T& rows, const TKey& key) { - Y_ENSURE(!rows.empty()); - auto it = rows.begin(); - - while (it != rows.end() && !(key < it->first)) { - ++it; - } - - return it; - } - - static auto Add(TRows& emplaceTo, TKey&& key, TString&& record) { - auto it = FindPartition(emplaceTo, key); - Y_ENSURE(it != emplaceTo.end()); - - auto& rows = it->second; - return std::make_pair(it, rows.Add(std::move(key), std::move(record))); - } - - static void RemoveFromSizeTracker(TRowsBy& container, ui64 prevSize, TRows::iterator rowIt) { - auto bucket = container.find(prevSize); - Y_ENSURE(bucket != container.end()); - - auto it = bucket->second.find(rowIt); - Y_ENSURE(it != bucket->second.end()); - - bucket->second.erase(it); - - if (bucket->second.empty()) { - container.erase(bucket); - } - } - -public: - explicit TTableRows(const TVector<TKeyRange>& keyRanges) - : ByPartition(MakeEmptyRows(MakeSplitPoints(keyRanges))) - , MemSize(0) - { - } - - bool CanAdd(const TKey& key, const TString& record, ui64 memLimit, ui64 batchSize) const { - if (MemSize + key.MemSize() + record.size() >= memLimit) { - return MemSize == 0; - } - - auto it = FindPartition(ByPartition, key); - Y_ENSURE(it != ByPartition.end()); - - if (it->second.RecordsSize() + record.size() >= batchSize) { - return it->second.RecordsSize() == 0; - } - - return true; - } - - void Add(TKey&& key, TString&& record) { - const ui64 recordSize = record.size(); - const ui64 memSize = key.MemSize() + recordSize; - - auto ret = Add(ByPartition, std::move(key), std::move(record)); - - if (ret.second) { - auto update = [it = ret.first](TRowsBy& container, ui64 prevSize, ui64 newSize) { - if (prevSize) { - RemoveFromSizeTracker(container, prevSize, it); - } - - Y_ENSURE(container[newSize].insert(it).second); - }; - - const auto& rows = ret.first->second; - update(ByMemSize, (rows.MemSize() - memSize), rows.MemSize()); - update(ByRecordsSize, (rows.RecordsSize() - AdjustedRecordSize(recordSize)), rows.RecordsSize()); - - MemSize += memSize; - } - } - - void Reshard(const TVector<TKeyRange>& keyRanges) { - auto newByPartition = MakeEmptyRows(MakeSplitPoints(keyRanges)); - - for (auto& [_, rows] : ByPartition) { - while (!rows.Empty()) { - auto handle = rows.Pop(); - Y_ENSURE(Add(newByPartition, std::move(handle.key()), std::move(handle.mapped())).second); - } - } - - ByMemSize.clear(); - ByRecordsSize.clear(); - ByPartition = std::move(newByPartition); - - for (auto it = ByPartition.begin(); it != ByPartition.end(); ++it) { - Y_ENSURE(ByMemSize[it->second.MemSize()].insert(it).second); - Y_ENSURE(ByRecordsSize[it->second.RecordsSize()].insert(it).second); - } - } - - bool HasData(ui64 memLimit, ui64 batchSize, bool force = false) const { - if (ByMemSize.empty()) { - Y_ENSURE(ByRecordsSize.empty()); - return false; - } - - Y_ENSURE(!ByMemSize.begin()->second.empty()); - Y_ENSURE(!ByRecordsSize.begin()->second.empty()); - - if (MemSize >= memLimit) { - return true; - } - - auto it = ByRecordsSize.begin()->second.begin(); - auto& rows = (*it)->second; - - if (rows.RecordsSize() >= batchSize || (force && !rows.Empty())) { - return true; - } - - return false; - } - - TString GetData(ui64 memLimit, ui64 batchSize, bool force = false) { - Y_ENSURE(HasData(memLimit, batchSize, force)); - Y_ENSURE(!ByMemSize.empty()); - Y_ENSURE(!ByRecordsSize.empty()); - Y_ENSURE(!ByMemSize.begin()->second.empty()); - Y_ENSURE(!ByRecordsSize.begin()->second.empty()); - - auto get = [this, batchSize](TRowsBy& from) { - auto it = from.begin()->second.begin(); - auto& rows = (*it)->second; - - RemoveFromSizeTracker(ByMemSize, rows.MemSize(), *it); - RemoveFromSizeTracker(ByRecordsSize, rows.RecordsSize(), *it); - - MemSize -= rows.MemSize(); - auto ret = rows.Serialize(batchSize); - MemSize += rows.MemSize(); - - if (rows.MemSize()) { - Y_ENSURE(ByMemSize[rows.MemSize()].insert(*it).second); - } - if (rows.RecordsSize()) { - Y_ENSURE(ByRecordsSize[rows.RecordsSize()].insert(*it).second); - } - - return ret; - }; - - if (MemSize >= memLimit) { - return get(ByMemSize); - } - - return get(ByRecordsSize); - } - -private: - TRows ByPartition; - TRowsBy ByMemSize; - TRowsBy ByRecordsSize; - - ui64 MemSize; - -}; // TTableRows - -class TDataAccumulator: public NPrivate::IDataAccumulator { -public: - explicit TDataAccumulator( - const TTableDescription& dumpedDesc, - const TTableDescription& actualDesc, - const TRestoreSettings& settings) - : KeyBuilder(dumpedDesc.GetColumns(), dumpedDesc.GetPrimaryKeyColumns()) - , Rows(actualDesc.GetKeyRanges()) - , MemLimit(settings.MemLimit_) - , BatchSize(settings.BytesPerRequest_) - { - } - - bool Fits(const TString& line) const override { - TGuard<TMutex> lock(Mutex); - return Rows.CanAdd(KeyBuilder.Build(line), line, MemLimit, BatchSize); - } - - void Feed(TString&& line) override { - TGuard<TMutex> lock(Mutex); - Rows.Add(KeyBuilder.Build(line), std::move(line)); - } - - bool Ready(bool force) const override { - TGuard<TMutex> lock(Mutex); - return Rows.HasData(MemLimit, BatchSize, force); - } - - TString GetData(bool force) override { - TGuard<TMutex> lock(Mutex); - return Rows.GetData(MemLimit, BatchSize, force); - } - - void Reshard(const TVector<TKeyRange>& keyRanges) { - TGuard<TMutex> lock(Mutex); - Rows.Reshard(keyRanges); - } - -private: - const TKeyBuilder KeyBuilder; - TTableRows Rows; - - const ui64 MemLimit; - const ui64 BatchSize; - - TMutex Mutex; - -}; // TDataAccumulator - -class TDataWriter: public NPrivate::IDataWriter { - static auto MakeSettings(const TRestoreSettings& settings, const TTableDescription& desc) { - auto importDataSettings = TImportYdbDumpDataSettings(settings) - .RequestType(DOC_API_REQUEST_TYPE); - - for (const auto& column : desc.GetColumns()) { - importDataSettings.AppendColumns(column.Name); - } - - return importDataSettings; - } - - bool Write(const TString& data) { - const ui32 maxRetries = 10; - TDuration retrySleep = TDuration::MilliSeconds(500); - - for (ui32 retryNumber = 0; retryNumber <= maxRetries; ++retryNumber) { + TInf, + TNull, + TString, + bool, + ui8, + i32, + ui32, + i64, + ui64 + > Value; + +}; // TValue + +template <typename TParser> +class TValueConverter { + TValue Convert(EPrimitiveType type) const { + switch (type) { + case EPrimitiveType::Bool: + return TValue(Parser.GetBool()); + case EPrimitiveType::Uint8: + return TValue(Parser.GetUint8()); + case EPrimitiveType::Int32: + return TValue(Parser.GetInt32()); + case EPrimitiveType::Uint32: + return TValue(Parser.GetUint32()); + case EPrimitiveType::Int64: + return TValue(Parser.GetInt64()); + case EPrimitiveType::Uint64: + return TValue(Parser.GetUint64()); + case EPrimitiveType::DyNumber: + return TValue(Parser.GetDyNumber()); + case EPrimitiveType::Date: + return TValue(Parser.GetDate().GetValue()); + case EPrimitiveType::Datetime: + return TValue(Parser.GetDatetime().GetValue()); + case EPrimitiveType::Timestamp: + return TValue(Parser.GetTimestamp().GetValue()); + case EPrimitiveType::String: + return TValue(Parser.GetString()); + case EPrimitiveType::Utf8: + return TValue(Parser.GetUtf8()); + default: + Y_ENSURE(false, "Unexpected primitive type: " << type); + } + } + + void Convert(TVector<TValue>& value) { + switch (Parser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + value.emplace_back(Convert(Parser.GetPrimitiveType())); + break; + + case TTypeParser::ETypeKind::Optional: + Parser.OpenOptional(); + if (Parser.IsNull()) { + value.emplace_back(TValue::Null()); + } else { + Convert(value); + } + Parser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::Tuple: + Parser.OpenTuple(); + while (Parser.TryNextElement()) { + Convert(value); + } + Parser.CloseTuple(); + break; + + default: + Y_ENSURE(false, "Unexpected type kind: " << Parser.GetKind()); + } + } + +public: + explicit TValueConverter(TParser& parser) + : Parser(parser) + { + } + + TVector<TValue> ConvertAll() { + TVector<TValue> value; + Convert(value); + return value; + } + + TValue ConvertSingle() { + auto value = ConvertAll(); + Y_ENSURE(value.size() == 1, "Unexpected value size: " << value.size()); + return value[0]; + } + +private: + TParser& Parser; + +}; // TValueConverter + +class TYdbDumpValueParser { + TString CheckedUnescape() const { + Y_ENSURE(Value.size() >= 2 || Value.front() == '"' || Value.back() == '"'); + return CGIUnescapeRet(TStringBuf(Value).Skip(1).Chop(1)); + } + +public: + explicit TYdbDumpValueParser(TStringBuf value, EPrimitiveType type) + : Value(value) + , Type(type) + , OptionalOpened(false) + { + } + + TTypeParser::ETypeKind GetKind() const { + return OptionalOpened + ? TTypeParser::ETypeKind::Primitive + : TTypeParser::ETypeKind::Optional; + } + + EPrimitiveType GetPrimitiveType() const { + return Type; + } + + bool GetBool() const { + return FromString<bool>(Value); + } + + ui8 GetUint8() const { + return FromString<ui8>(Value); + } + + i32 GetInt32() const { + return FromString<i32>(Value); + } + + ui32 GetUint32() const { + return FromString<ui32>(Value); + } + + i64 GetInt64() const { + return FromString<i64>(Value); + } + + ui64 GetUint64() const { + return FromString<ui64>(Value); + } + + TString GetDyNumber() const { + return ToString(Value); + } + + TInstant GetDate() const { + return TInstant::ParseIso8601(Value); + } + + TInstant GetDatetime() const { + return TInstant::ParseIso8601(Value); + } + + TInstant GetTimestamp() const { + return TInstant::ParseIso8601(Value); + } + + TString GetString() const { + return CheckedUnescape(); + } + + TString GetUtf8() const { + return CheckedUnescape(); + } + + bool IsNull() const { + return Value == "null"; + } + + void OpenOptional() { + OptionalOpened = true; + } + + void CloseOptional() { + OptionalOpened = false; + } + + bool TryNextElement() { + return false; + } + + void OpenTuple() {} + void CloseTuple() {} + +private: + const TStringBuf Value; + const EPrimitiveType Type; + + bool OptionalOpened; + +}; // TYdbDumpValueParser + +class TKey: public TVector<TValue> { +public: + using TVector<TValue>::TVector; + + TKey(TVector<TValue>&& values) + : TVector<TValue>(std::move(values)) + { + } + + bool operator<(const TKey& rhs) const { + for (ui32 i = 0; i < Min(size(), rhs.size()); ++i) { + if (at(i) < rhs.at(i)) { + return true; + } + + if (rhs.at(i) < at(i)) { + return false; + } + } + + return size() < rhs.size(); + } + + ui64 MemSize() const { + return Accumulate(begin(), end(), ui64(0), [](ui64 s, const auto& v) { + return s + v.MemSize(); + }); + } + +}; // TKey + +using TSplitPoint = TKey; + +class TKeyBuilder { + static auto MakeKeyColumnIds(const TVector<TString>& keyColumns) { + THashMap<TString, ui32> keyColumnIds; + + for (ui32 i = 0; i < keyColumns.size(); ++i) { + Y_ENSURE(keyColumnIds.emplace(keyColumns.at(i), i).second); + } + + return keyColumnIds; + } + + static EPrimitiveType GetPrimitiveType(TTypeParser& parser) { + switch (parser.GetKind()) { + case TTypeParser::ETypeKind::Optional: + parser.OpenOptional(); + return GetPrimitiveType(parser); + case TTypeParser::ETypeKind::Primitive: + return parser.GetPrimitive(); + default: + Y_ENSURE(false, "Unexpected type kind: " << parser.GetKind()); + } + } + + static EPrimitiveType GetPrimitiveType(const TType& type) { + TTypeParser parser(type); + return GetPrimitiveType(parser); + } + +public: + explicit TKeyBuilder(const TVector<TColumn>& columns, const TVector<TString>& keyColumns) + : Columns(columns) + , KeyColumnIds(MakeKeyColumnIds(keyColumns)) + { + } + + TKey Build(TStringBuf line) const { + TMap<ui32, TValue> values; + + const TStringBuf origLine = line; + for (const auto& column : Columns) { + TStringBuf value = line.NextTok(','); + Y_ENSURE(value, "Empty token on line: " << origLine); + + auto it = KeyColumnIds.find(column.Name); + if (it == KeyColumnIds.end()) { + continue; + } + + TYdbDumpValueParser parser(value, GetPrimitiveType(column.Type)); + values.emplace(it->second, TValueConverter<TYdbDumpValueParser>(parser).ConvertSingle()); + } + + TKey key; + for (auto& [_, value] : values) { + key.push_back(std::move(value)); + } + + return key; + } + +private: + const TVector<TColumn> Columns; + const THashMap<TString, ui32> KeyColumnIds; + +}; // TKeyBuilder + +ui64 AdjustedRecordSize(ui64 recordSize) { + return recordSize + 1 /* \n */; +} + +class TPartitionRows { + void IncSize(ui64 keySize, ui64 recordSize) { + MemSizeCounter += (keySize + recordSize); + RecordsSizeCounter += AdjustedRecordSize(recordSize); + } + + void DecSize(ui64 keySize, ui64 recordSize) { + MemSizeCounter -= (keySize + recordSize); + RecordsSizeCounter -= AdjustedRecordSize(recordSize); + } + +public: + TPartitionRows() + : MemSizeCounter(0) + , RecordsSizeCounter(0) + { + } + + ui64 MemSize() const { + return MemSizeCounter; + } + + ui64 RecordsSize() const { + return RecordsSizeCounter; + } + + bool Add(TKey&& key, TString&& record) { + auto ret = Rows.emplace(std::move(key), std::move(record)); + if (!ret.second) { + return ret.second; + } + + IncSize(ret.first->first.MemSize(), ret.first->second.size()); + return ret.second; + } + + bool Empty() const { + return Rows.empty(); + } + + auto Pop() { + Y_ENSURE(!Empty()); + auto ret = Rows.extract(Rows.begin()); + + DecSize(ret.key().MemSize(), ret.mapped().size()); + return ret; + } + + TString Serialize(ui64 maxSize) { + TStringBuilder result; + + if (Empty()) { + return result; + } + + auto handle = Pop(); + while (result.size() + handle.mapped().size() < maxSize) { + result << handle.mapped() << "\n"; + + if (Empty()) { + return result; + } + + handle = Pop(); + } + + Add(std::move(handle.key()), std::move(handle.mapped())); + return result; + } + +private: + TMap<TKey, TString> Rows; + ui64 MemSizeCounter; + ui64 RecordsSizeCounter; + +}; // TPartitionRows + +class TTableRows { + template <typename TIterator> + struct TIteratorHash { + size_t operator()(const TIterator& x) const { + return THash<intptr_t>()(reinterpret_cast<intptr_t>(&*x)); + } + }; + + // key is a right-side boundary of partition + using TRows = TMap<TSplitPoint, TPartitionRows>; + using TRowsBy = TMap<ui64, THashSet<TRows::iterator, TIteratorHash<TRows::iterator>>, TGreater<ui64>>; + + static auto MakeSplitPoints(const TVector<TKeyRange>& keyRanges) { + Y_ENSURE(!keyRanges.empty()); + + TVector<TSplitPoint> splitPoints; + + auto it = keyRanges.begin(); + while (++it != keyRanges.end()) { + const auto& from = it->From(); + + Y_ENSURE(from.Defined()); + Y_ENSURE(from->IsInclusive()); + + TValueParser parser(from->GetValue()); + splitPoints.push_back(TValueConverter<TValueParser>(parser).ConvertAll()); + } + + return splitPoints; + } + + static auto MakeEmptyRows(TVector<TSplitPoint>&& splitPoints) { + TRows rows; + + for (auto& splitPoint : splitPoints) { + rows.emplace(std::move(splitPoint), TPartitionRows()); + } + + // last boundary is +inf + rows.emplace(TSplitPoint{TValue::Inf()}, TPartitionRows()); + + return rows; + } + + template <typename T> + static auto FindPartition(T& rows, const TKey& key) { + Y_ENSURE(!rows.empty()); + auto it = rows.begin(); + + while (it != rows.end() && !(key < it->first)) { + ++it; + } + + return it; + } + + static auto Add(TRows& emplaceTo, TKey&& key, TString&& record) { + auto it = FindPartition(emplaceTo, key); + Y_ENSURE(it != emplaceTo.end()); + + auto& rows = it->second; + return std::make_pair(it, rows.Add(std::move(key), std::move(record))); + } + + static void RemoveFromSizeTracker(TRowsBy& container, ui64 prevSize, TRows::iterator rowIt) { + auto bucket = container.find(prevSize); + Y_ENSURE(bucket != container.end()); + + auto it = bucket->second.find(rowIt); + Y_ENSURE(it != bucket->second.end()); + + bucket->second.erase(it); + + if (bucket->second.empty()) { + container.erase(bucket); + } + } + +public: + explicit TTableRows(const TVector<TKeyRange>& keyRanges) + : ByPartition(MakeEmptyRows(MakeSplitPoints(keyRanges))) + , MemSize(0) + { + } + + bool CanAdd(const TKey& key, const TString& record, ui64 memLimit, ui64 batchSize) const { + if (MemSize + key.MemSize() + record.size() >= memLimit) { + return MemSize == 0; + } + + auto it = FindPartition(ByPartition, key); + Y_ENSURE(it != ByPartition.end()); + + if (it->second.RecordsSize() + record.size() >= batchSize) { + return it->second.RecordsSize() == 0; + } + + return true; + } + + void Add(TKey&& key, TString&& record) { + const ui64 recordSize = record.size(); + const ui64 memSize = key.MemSize() + recordSize; + + auto ret = Add(ByPartition, std::move(key), std::move(record)); + + if (ret.second) { + auto update = [it = ret.first](TRowsBy& container, ui64 prevSize, ui64 newSize) { + if (prevSize) { + RemoveFromSizeTracker(container, prevSize, it); + } + + Y_ENSURE(container[newSize].insert(it).second); + }; + + const auto& rows = ret.first->second; + update(ByMemSize, (rows.MemSize() - memSize), rows.MemSize()); + update(ByRecordsSize, (rows.RecordsSize() - AdjustedRecordSize(recordSize)), rows.RecordsSize()); + + MemSize += memSize; + } + } + + void Reshard(const TVector<TKeyRange>& keyRanges) { + auto newByPartition = MakeEmptyRows(MakeSplitPoints(keyRanges)); + + for (auto& [_, rows] : ByPartition) { + while (!rows.Empty()) { + auto handle = rows.Pop(); + Y_ENSURE(Add(newByPartition, std::move(handle.key()), std::move(handle.mapped())).second); + } + } + + ByMemSize.clear(); + ByRecordsSize.clear(); + ByPartition = std::move(newByPartition); + + for (auto it = ByPartition.begin(); it != ByPartition.end(); ++it) { + Y_ENSURE(ByMemSize[it->second.MemSize()].insert(it).second); + Y_ENSURE(ByRecordsSize[it->second.RecordsSize()].insert(it).second); + } + } + + bool HasData(ui64 memLimit, ui64 batchSize, bool force = false) const { + if (ByMemSize.empty()) { + Y_ENSURE(ByRecordsSize.empty()); + return false; + } + + Y_ENSURE(!ByMemSize.begin()->second.empty()); + Y_ENSURE(!ByRecordsSize.begin()->second.empty()); + + if (MemSize >= memLimit) { + return true; + } + + auto it = ByRecordsSize.begin()->second.begin(); + auto& rows = (*it)->second; + + if (rows.RecordsSize() >= batchSize || (force && !rows.Empty())) { + return true; + } + + return false; + } + + TString GetData(ui64 memLimit, ui64 batchSize, bool force = false) { + Y_ENSURE(HasData(memLimit, batchSize, force)); + Y_ENSURE(!ByMemSize.empty()); + Y_ENSURE(!ByRecordsSize.empty()); + Y_ENSURE(!ByMemSize.begin()->second.empty()); + Y_ENSURE(!ByRecordsSize.begin()->second.empty()); + + auto get = [this, batchSize](TRowsBy& from) { + auto it = from.begin()->second.begin(); + auto& rows = (*it)->second; + + RemoveFromSizeTracker(ByMemSize, rows.MemSize(), *it); + RemoveFromSizeTracker(ByRecordsSize, rows.RecordsSize(), *it); + + MemSize -= rows.MemSize(); + auto ret = rows.Serialize(batchSize); + MemSize += rows.MemSize(); + + if (rows.MemSize()) { + Y_ENSURE(ByMemSize[rows.MemSize()].insert(*it).second); + } + if (rows.RecordsSize()) { + Y_ENSURE(ByRecordsSize[rows.RecordsSize()].insert(*it).second); + } + + return ret; + }; + + if (MemSize >= memLimit) { + return get(ByMemSize); + } + + return get(ByRecordsSize); + } + +private: + TRows ByPartition; + TRowsBy ByMemSize; + TRowsBy ByRecordsSize; + + ui64 MemSize; + +}; // TTableRows + +class TDataAccumulator: public NPrivate::IDataAccumulator { +public: + explicit TDataAccumulator( + const TTableDescription& dumpedDesc, + const TTableDescription& actualDesc, + const TRestoreSettings& settings) + : KeyBuilder(dumpedDesc.GetColumns(), dumpedDesc.GetPrimaryKeyColumns()) + , Rows(actualDesc.GetKeyRanges()) + , MemLimit(settings.MemLimit_) + , BatchSize(settings.BytesPerRequest_) + { + } + + bool Fits(const TString& line) const override { + TGuard<TMutex> lock(Mutex); + return Rows.CanAdd(KeyBuilder.Build(line), line, MemLimit, BatchSize); + } + + void Feed(TString&& line) override { + TGuard<TMutex> lock(Mutex); + Rows.Add(KeyBuilder.Build(line), std::move(line)); + } + + bool Ready(bool force) const override { + TGuard<TMutex> lock(Mutex); + return Rows.HasData(MemLimit, BatchSize, force); + } + + TString GetData(bool force) override { + TGuard<TMutex> lock(Mutex); + return Rows.GetData(MemLimit, BatchSize, force); + } + + void Reshard(const TVector<TKeyRange>& keyRanges) { + TGuard<TMutex> lock(Mutex); + Rows.Reshard(keyRanges); + } + +private: + const TKeyBuilder KeyBuilder; + TTableRows Rows; + + const ui64 MemLimit; + const ui64 BatchSize; + + TMutex Mutex; + +}; // TDataAccumulator + +class TDataWriter: public NPrivate::IDataWriter { + static auto MakeSettings(const TRestoreSettings& settings, const TTableDescription& desc) { + auto importDataSettings = TImportYdbDumpDataSettings(settings) + .RequestType(DOC_API_REQUEST_TYPE); + + for (const auto& column : desc.GetColumns()) { + importDataSettings.AppendColumns(column.Name); + } + + return importDataSettings; + } + + bool Write(const TString& data) { + const ui32 maxRetries = 10; + TDuration retrySleep = TDuration::MilliSeconds(500); + + for (ui32 retryNumber = 0; retryNumber <= maxRetries; ++retryNumber) { while (!RequestLimiter.IsAvail()) { Sleep(Min(TDuration::MicroSeconds(RequestLimiter.GetWaitTime()), RateLimiterSettings.ReactionTime_)); - if (IsStopped()) { - return false; - } - } - - if (IsStopped()) { - return false; - } - + if (IsStopped()) { + return false; + } + } + + if (IsStopped()) { + return false; + } + RequestLimiter.Use(1); - auto importResult = ImportClient.ImportData(Path, data, Settings).GetValueSync(); - - if (importResult.IsSuccess()) { - return true; - } - - if (retryNumber == maxRetries) { - return false; - } - - switch (importResult.GetStatus()) { - case EStatus::PRECONDITION_FAILED: { - // table's partitioning has been changed - TMaybe<TTableDescription> desc; - auto descResult = DescribeTable(TableClient, Path, desc); - if (!descResult.IsSuccess()) { - return false; - } - - Accumulator->Reshard(desc->GetKeyRanges()); - - TStringInput input(data); - TString line; - - while (input.ReadLine(line)) { - Accumulator->Feed(std::move(line)); - } - - break; - } - - case EStatus::ABORTED: - break; - - case EStatus::OVERLOADED: - case EStatus::CLIENT_RESOURCE_EXHAUSTED: - ExponentialBackoff(retrySleep); - break; - - case EStatus::UNAVAILABLE: - ExponentialBackoff(retrySleep); - break; - - case EStatus::TRANSPORT_UNAVAILABLE: - ExponentialBackoff(retrySleep); - break; - - default: - return false; - } - } - - return false; - } - - void Stop() { - AtomicSet(Stopped, 1); - } - - bool IsStopped() const { - return AtomicGet(Stopped) == 1; - } - -public: - explicit TDataWriter( - const TString& path, - const TTableDescription& desc, - const TRestoreSettings& settings, - TImportClient& importClient, - TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator) - : Path(path) - , Settings(MakeSettings(settings, desc)) - , ImportClient(importClient) - , TableClient(tableClient) - , Accumulator(dynamic_cast<TDataAccumulator*>(accumulator)) - , RateLimiterSettings(settings.RateLimiterSettings_) + auto importResult = ImportClient.ImportData(Path, data, Settings).GetValueSync(); + + if (importResult.IsSuccess()) { + return true; + } + + if (retryNumber == maxRetries) { + return false; + } + + switch (importResult.GetStatus()) { + case EStatus::PRECONDITION_FAILED: { + // table's partitioning has been changed + TMaybe<TTableDescription> desc; + auto descResult = DescribeTable(TableClient, Path, desc); + if (!descResult.IsSuccess()) { + return false; + } + + Accumulator->Reshard(desc->GetKeyRanges()); + + TStringInput input(data); + TString line; + + while (input.ReadLine(line)) { + Accumulator->Feed(std::move(line)); + } + + break; + } + + case EStatus::ABORTED: + break; + + case EStatus::OVERLOADED: + case EStatus::CLIENT_RESOURCE_EXHAUSTED: + ExponentialBackoff(retrySleep); + break; + + case EStatus::UNAVAILABLE: + ExponentialBackoff(retrySleep); + break; + + case EStatus::TRANSPORT_UNAVAILABLE: + ExponentialBackoff(retrySleep); + break; + + default: + return false; + } + } + + return false; + } + + void Stop() { + AtomicSet(Stopped, 1); + } + + bool IsStopped() const { + return AtomicGet(Stopped) == 1; + } + +public: + explicit TDataWriter( + const TString& path, + const TTableDescription& desc, + const TRestoreSettings& settings, + TImportClient& importClient, + TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator) + : Path(path) + , Settings(MakeSettings(settings, desc)) + , ImportClient(importClient) + , TableClient(tableClient) + , Accumulator(dynamic_cast<TDataAccumulator*>(accumulator)) + , RateLimiterSettings(settings.RateLimiterSettings_) , RequestLimiter(RateLimiterSettings.GetRps(), RateLimiterSettings.GetRps()) - { - Y_ENSURE(Accumulator); - - TasksQueue = MakeHolder<TThreadPool>(TThreadPool::TParams().SetBlocking(true).SetCatching(true)); - TasksQueue->Start(settings.InFly_, settings.InFly_ + 1); - } - - bool Push(TString&& data) override { - Y_ENSURE(data.size() < TRestoreSettings::MaxBytesPerRequest, "Data is too long"); - - if (IsStopped()) { - return false; - } - - auto func = [this, data = std::move(data)]() { - if (!Write(data)) { - Stop(); - } - }; - - return TasksQueue->AddFunc(func); - } - - void Wait() override { - TasksQueue->Stop(); - } - -private: - const TString Path; - const TImportYdbDumpDataSettings Settings; - TImportClient& ImportClient; - TTableClient& TableClient; - TDataAccumulator* Accumulator; - - const TRateLimiterSettings RateLimiterSettings; + { + Y_ENSURE(Accumulator); + + TasksQueue = MakeHolder<TThreadPool>(TThreadPool::TParams().SetBlocking(true).SetCatching(true)); + TasksQueue->Start(settings.InFly_, settings.InFly_ + 1); + } + + bool Push(TString&& data) override { + Y_ENSURE(data.size() < TRestoreSettings::MaxBytesPerRequest, "Data is too long"); + + if (IsStopped()) { + return false; + } + + auto func = [this, data = std::move(data)]() { + if (!Write(data)) { + Stop(); + } + }; + + return TasksQueue->AddFunc(func); + } + + void Wait() override { + TasksQueue->Stop(); + } + +private: + const TString Path; + const TImportYdbDumpDataSettings Settings; + TImportClient& ImportClient; + TTableClient& TableClient; + TDataAccumulator* Accumulator; + + const TRateLimiterSettings RateLimiterSettings; using TRpsLimiter = TBucketQuoter<ui64>; - TRpsLimiter RequestLimiter; - - THolder<IThreadPool> TasksQueue; - TAtomic Stopped; - -}; // TDataWriter - -} // anonymous - -NPrivate::IDataAccumulator* CreateImportDataAccumulator( - const NTable::TTableDescription& dumpedDesc, - const NTable::TTableDescription& actualDesc, - const TRestoreSettings& settings) { - return new TDataAccumulator(dumpedDesc, actualDesc, settings); -} - -NPrivate::IDataWriter* CreateImportDataWriter( - const TString& path, - const TTableDescription& desc, - TImportClient& importClient, - TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator, - const TRestoreSettings& settings) { - return new TDataWriter(path, desc, settings, importClient, tableClient, accumulator); -} - -} // NDump -} // NYdb + TRpsLimiter RequestLimiter; + + THolder<IThreadPool> TasksQueue; + TAtomic Stopped; + +}; // TDataWriter + +} // anonymous + +NPrivate::IDataAccumulator* CreateImportDataAccumulator( + const NTable::TTableDescription& dumpedDesc, + const NTable::TTableDescription& actualDesc, + const TRestoreSettings& settings) { + return new TDataAccumulator(dumpedDesc, actualDesc, settings); +} + +NPrivate::IDataWriter* CreateImportDataWriter( + const TString& path, + const TTableDescription& desc, + TImportClient& importClient, + TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator, + const TRestoreSettings& settings) { + return new TDataWriter(path, desc, settings, importClient, tableClient, accumulator); +} + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/restore_import_data.h b/ydb/public/lib/ydb_cli/dump/restore_import_data.h index 32b552c7896..93ff09bf221 100644 --- a/ydb/public/lib/ydb_cli/dump/restore_import_data.h +++ b/ydb/public/lib/ydb_cli/dump/restore_import_data.h @@ -1,22 +1,22 @@ -#pragma once - -#include "restore_impl.h" - -namespace NYdb { -namespace NDump { - -NPrivate::IDataAccumulator* CreateImportDataAccumulator( - const NTable::TTableDescription& dumpedDesc, - const NTable::TTableDescription& actualDesc, - const TRestoreSettings& settings); - -NPrivate::IDataWriter* CreateImportDataWriter( - const TString& path, - const NTable::TTableDescription& desc, - NImport::TImportClient& importClient, - NTable::TTableClient& tableClient, - NPrivate::IDataAccumulator* accumulator, - const TRestoreSettings& settings); - -} // NDump -} // NYdb +#pragma once + +#include "restore_impl.h" + +namespace NYdb { +namespace NDump { + +NPrivate::IDataAccumulator* CreateImportDataAccumulator( + const NTable::TTableDescription& dumpedDesc, + const NTable::TTableDescription& actualDesc, + const TRestoreSettings& settings); + +NPrivate::IDataWriter* CreateImportDataWriter( + const TString& path, + const NTable::TTableDescription& desc, + NImport::TImportClient& importClient, + NTable::TTableClient& tableClient, + NPrivate::IDataAccumulator* accumulator, + const TRestoreSettings& settings); + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/util/util.cpp b/ydb/public/lib/ydb_cli/dump/util/util.cpp index 332cb0e736f..37c8d4b646b 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.cpp +++ b/ydb/public/lib/ydb_cli/dump/util/util.cpp @@ -1,17 +1,17 @@ -#include "util.h" - +#include "util.h" + #include <ydb/public/lib/ydb_cli/common/recursive_list.h> #include <util/folder/path.h> -namespace NYdb { -namespace NDump { - -using namespace NScheme; -using namespace NTable; - +namespace NYdb { +namespace NDump { + +using namespace NScheme; +using namespace NTable; + namespace { - + // function must return TStatus or derived struct // function must be re-invokable template <typename TFunction> @@ -50,55 +50,55 @@ decltype(auto) RetryFunction( // unreachable return func(); } - + } // anonymous -TStatus DescribeTable(TTableClient& tableClient, const TString& path, TMaybe<TTableDescription>& out) { - auto func = [&path, &out](TSession session) { - auto settings = TDescribeTableSettings().WithKeyShardBoundary(true); - auto result = session.DescribeTable(path, settings).GetValueSync(); - - if (result.IsSuccess()) { - out = result.GetTableDescription(); - } - - return result; - }; - - return tableClient.RetryOperationSync(func, TRetryOperationSettings().Idempotent(true)); -} - -void ExponentialBackoff(TDuration& sleep, TDuration max) { - Sleep(sleep); - sleep = Min(sleep * 2, max); -} - -TDescribePathResult DescribePath(TSchemeClient& schemeClient, const TString& path, const TDescribePathSettings& settings) { - return RetryFunction([&]() -> TDescribePathResult { +TStatus DescribeTable(TTableClient& tableClient, const TString& path, TMaybe<TTableDescription>& out) { + auto func = [&path, &out](TSession session) { + auto settings = TDescribeTableSettings().WithKeyShardBoundary(true); + auto result = session.DescribeTable(path, settings).GetValueSync(); + + if (result.IsSuccess()) { + out = result.GetTableDescription(); + } + + return result; + }; + + return tableClient.RetryOperationSync(func, TRetryOperationSettings().Idempotent(true)); +} + +void ExponentialBackoff(TDuration& sleep, TDuration max) { + Sleep(sleep); + sleep = Min(sleep * 2, max); +} + +TDescribePathResult DescribePath(TSchemeClient& schemeClient, const TString& path, const TDescribePathSettings& settings) { + return RetryFunction([&]() -> TDescribePathResult { return schemeClient.DescribePath(path, settings).GetValueSync(); }); } -TStatus MakeDirectory(TSchemeClient& schemeClient, const TString& path, const TMakeDirectorySettings& settings) { +TStatus MakeDirectory(TSchemeClient& schemeClient, const TString& path, const TMakeDirectorySettings& settings) { + return RetryFunction([&]() -> TStatus { + return schemeClient.MakeDirectory(path, settings).GetValueSync(); + }); +} + +TStatus RemoveDirectory(TSchemeClient& schemeClient, const TString& path, const TRemoveDirectorySettings& settings) { return RetryFunction([&]() -> TStatus { - return schemeClient.MakeDirectory(path, settings).GetValueSync(); - }); -} - -TStatus RemoveDirectory(TSchemeClient& schemeClient, const TString& path, const TRemoveDirectorySettings& settings) { - return RetryFunction([&]() -> TStatus { return schemeClient.RemoveDirectory(path, settings).GetValueSync(); }); } TStatus RemoveDirectoryRecursive( - TTableClient& tableClient, - TSchemeClient& schemeClient, + TTableClient& tableClient, + TSchemeClient& schemeClient, const TString& path, - const TRemoveDirectorySettings& settings, + const TRemoveDirectorySettings& settings, bool removeSelf) { - auto recursiveListResult = NConsoleClient::RecursiveList(schemeClient, path, {}, removeSelf); + auto recursiveListResult = NConsoleClient::RecursiveList(schemeClient, path, {}, removeSelf); if (!recursiveListResult.Status.IsSuccess()) { return recursiveListResult.Status; } @@ -108,17 +108,17 @@ TStatus RemoveDirectoryRecursive( for (auto it = recursiveListResult.Entries.rbegin(); it != recursiveListResult.Entries.rend(); ++it) { const auto& entry = *it; switch (entry.Type) { - case ESchemeEntryType::Directory: { - auto result = RemoveDirectory(schemeClient, entry.Name, settings); + case ESchemeEntryType::Directory: { + auto result = RemoveDirectory(schemeClient, entry.Name, settings); if (!result.IsSuccess()) { return result; } break; } - case ESchemeEntryType::Table: { - auto result = tableClient.RetryOperationSync([path = entry.Name](TSession session) { - return session.DropTable(path).GetValueSync(); - }); + case ESchemeEntryType::Table: { + auto result = tableClient.RetryOperationSync([path = entry.Name](TSession session) { + return session.DropTable(path).GetValueSync(); + }); if (!result.IsSuccess()) { return result; } @@ -132,5 +132,5 @@ TStatus RemoveDirectoryRecursive( return TStatus(EStatus::SUCCESS, {}); } -} // NDump -} // NYdb +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/util/util.h b/ydb/public/lib/ydb_cli/dump/util/util.h index c1c92f476d2..d5e8fc3b97d 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.h +++ b/ydb/public/lib/ydb_cli/dump/util/util.h @@ -1,53 +1,53 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - -#include <util/generic/maybe.h> - -namespace NYdb { -namespace NDump { - -template <typename TResult> -inline TResult Result(EStatus code = EStatus::SUCCESS, const TString& error = {}) { - NYql::TIssues issues; - if (error) { - issues.AddIssue(NYql::TIssue(error)); - } + +#include <util/generic/maybe.h> + +namespace NYdb { +namespace NDump { + +template <typename TResult> +inline TResult Result(EStatus code = EStatus::SUCCESS, const TString& error = {}) { + NYql::TIssues issues; + if (error) { + issues.AddIssue(NYql::TIssue(error)); + } return TResult(TStatus(code, std::move(issues))); -} - -template <typename TResult> -inline TResult Result(TStatus&& status) { - return TResult(std::move(status)); -} - -TStatus DescribeTable(NTable::TTableClient& tableClient, const TString& path, TMaybe<NTable::TTableDescription>& out); - -void ExponentialBackoff(TDuration& sleep, TDuration max = TDuration::Minutes(5)); - -NScheme::TDescribePathResult DescribePath( - NScheme::TSchemeClient& schemeClient, - const TString& path, - const NScheme::TDescribePathSettings& settings = {}); - -TStatus MakeDirectory( - NScheme::TSchemeClient& schemeClient, - const TString& path, - const NScheme::TMakeDirectorySettings& settings = {}); - -TStatus RemoveDirectory( - NScheme::TSchemeClient& schemeClient, - const TString& path, - const NScheme::TRemoveDirectorySettings& settings = {}); +} + +template <typename TResult> +inline TResult Result(TStatus&& status) { + return TResult(std::move(status)); +} + +TStatus DescribeTable(NTable::TTableClient& tableClient, const TString& path, TMaybe<NTable::TTableDescription>& out); + +void ExponentialBackoff(TDuration& sleep, TDuration max = TDuration::Minutes(5)); + +NScheme::TDescribePathResult DescribePath( + NScheme::TSchemeClient& schemeClient, + const TString& path, + const NScheme::TDescribePathSettings& settings = {}); + +TStatus MakeDirectory( + NScheme::TSchemeClient& schemeClient, + const TString& path, + const NScheme::TMakeDirectorySettings& settings = {}); + +TStatus RemoveDirectory( + NScheme::TSchemeClient& schemeClient, + const TString& path, + const NScheme::TRemoveDirectorySettings& settings = {}); TStatus RemoveDirectoryRecursive( - NTable::TTableClient& tableClient, - NScheme::TSchemeClient& schemeClient, - const TString& path, - const NScheme::TRemoveDirectorySettings& settings = {}, - bool removeSelf = true); - -} // NDump -} // NYdb + NTable::TTableClient& tableClient, + NScheme::TSchemeClient& schemeClient, + const TString& path, + const NScheme::TRemoveDirectorySettings& settings = {}, + bool removeSelf = true); + +} // NDump +} // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/util/ya.make b/ydb/public/lib/ydb_cli/dump/util/ya.make index 622e24572d5..24a3a5f2575 100644 --- a/ydb/public/lib/ydb_cli/dump/util/ya.make +++ b/ydb/public/lib/ydb_cli/dump/util/ya.make @@ -16,4 +16,4 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_types/status ) -END() +END() diff --git a/ydb/public/lib/ydb_cli/dump/ya.make b/ydb/public/lib/ydb_cli/dump/ya.make index 740768b9ff1..b85766bb396 100644 --- a/ydb/public/lib/ydb_cli/dump/ya.make +++ b/ydb/public/lib/ydb_cli/dump/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER( - ilnaz - g:kikimr -) - -SRCS( +LIBRARY() + +OWNER( + ilnaz + g:kikimr +) + +SRCS( dump.cpp dump_impl.cpp restore_impl.cpp restore_import_data.cpp restore_compat.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/bucket_quoter library/cpp/string_utils/quote ydb/library/backup @@ -21,6 +21,6 @@ PEERDIR( ydb/public/lib/ydb_cli/common ydb/public/lib/ydb_cli/dump/util ydb/public/sdk/cpp/client/ydb_proto -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 16ac4789e38..c43e9aab694 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -66,7 +66,7 @@ protected: promise.SetValue(std::move(st)); }; - Connections_->RunDeferred<TService, TRequest, TResponse>( + Connections_->RunDeferred<TService, TRequest, TResponse>( std::move(request), extractor, rpc, @@ -79,37 +79,37 @@ protected: return promise.GetFuture(); } - template<typename TService, typename TRequest, typename TResponse, typename TOp> - NThreading::TFuture<TOp> RunOperation( - TRequest&& request, - TAsyncRequest<TService, TRequest, TResponse> rpc, + template<typename TService, typename TRequest, typename TResponse, typename TOp> + NThreading::TFuture<TOp> RunOperation( + TRequest&& request, + TAsyncRequest<TService, TRequest, TResponse> rpc, const TRpcRequestSettings& requestSettings = {}, - TDuration timeout = TDuration::Zero()) - { - auto promise = NThreading::NewPromise<TOp>(); - + TDuration timeout = TDuration::Zero()) + { + auto promise = NThreading::NewPromise<TOp>(); + auto extractor = [promise] (Ydb::Operations::Operation* operation, TPlainStatus status) mutable { TStatus st(std::move(status)); - if (!operation) { - promise.SetValue(TOp(std::move(st))); - } else { - promise.SetValue(TOp(std::move(st), std::move(*operation))); - } - }; - - Connections_->RunDeferred<TService, TRequest, TResponse>( - std::move(request), - extractor, - rpc, - DbDriverState_, + if (!operation) { + promise.SetValue(TOp(std::move(st))); + } else { + promise.SetValue(TOp(std::move(st), std::move(*operation))); + } + }; + + Connections_->RunDeferred<TService, TRequest, TResponse>( + std::move(request), + extractor, + rpc, + DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, requestSettings, - timeout); - - return promise.GetFuture(); - } - + timeout); + + return promise.GetFuture(); + } + protected: std::shared_ptr<TGRpcConnectionsImpl> Connections_; TDbDriverStatePtr DbDriverState_; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.h b/ydb/public/sdk/cpp/client/ydb_driver/driver.h index 54438330a99..39c91d8852d 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.h +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.h @@ -8,7 +8,7 @@ #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> #include <library/cpp/logger/backend.h> - + //////////////////////////////////////////////////////////////////////////////// namespace NYdb { diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp index f12b927882c..c391c033e13 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp +++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp @@ -1,5 +1,5 @@ #include "export.h" - + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #undef INCLUDE_YDB_INTERNAL_H @@ -13,187 +13,187 @@ #include <google/protobuf/repeated_field.h> #include <google/protobuf/timestamp.pb.h> -namespace NYdb { -namespace NExport { - -using namespace NThreading; -using namespace Ydb::Export; - -/// Common -namespace { - -TInstant ProtoTimestampToInstant(const NProtoBuf::Timestamp& timestamp) { - ui64 us = timestamp.seconds() * 1000000; - us += timestamp.nanos() / 1000; - return TInstant::MicroSeconds(us); -} - -TVector<TExportItemProgress> ItemsProgressFromProto(const google::protobuf::RepeatedPtrField<ExportItemProgress>& proto) { - TVector<TExportItemProgress> result(Reserve(proto.size())); - - for (const auto& protoItem : proto) { - auto& item = result.emplace_back(); - item.PartsTotal = protoItem.parts_total(); - item.PartsCompleted = protoItem.parts_completed(); - item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); - item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); - } - - return result; -} - -} // anonymous - -/// YT -TExportToYtResponse::TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation) - : TOperation(std::move(status), std::move(operation)) -{ - ExportToYtMetadata metadata; - GetProto().metadata().UnpackTo(&metadata); - - // settings - Metadata_.Settings.Host(metadata.settings().host()); - Metadata_.Settings.Port(metadata.settings().port()); - Metadata_.Settings.Token(metadata.settings().token()); - - for (const auto& item : metadata.settings().items()) { - Metadata_.Settings.AppendItem({item.source_path(), item.destination_path()}); - } - - Metadata_.Settings.Description(metadata.settings().description()); - Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); - Metadata_.Settings.UseTypeV3(metadata.settings().use_type_v3()); - - // progress - Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); - Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); -} - -const TExportToYtResponse::TMetadata& TExportToYtResponse::Metadata() const { - return Metadata_; -} - -/// S3 -TExportToS3Response::TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) - : TOperation(std::move(status), std::move(operation)) -{ - ExportToS3Metadata metadata; - GetProto().metadata().UnpackTo(&metadata); - - // settings - Metadata_.Settings.Endpoint(metadata.settings().endpoint()); - Metadata_.Settings.Scheme(TProtoAccessor::FromProto<ExportToS3Settings>(metadata.settings().scheme())); - Metadata_.Settings.StorageClass(TProtoAccessor::FromProto(metadata.settings().storage_class())); - Metadata_.Settings.Bucket(metadata.settings().bucket()); - Metadata_.Settings.AccessKey(metadata.settings().access_key()); - Metadata_.Settings.SecretKey(metadata.settings().secret_key()); - - for (const auto& item : metadata.settings().items()) { - Metadata_.Settings.AppendItem({item.source_path(), item.destination_prefix()}); - } - - Metadata_.Settings.Description(metadata.settings().description()); - Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); - - // progress - Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); - Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); -} - -const TExportToS3Response::TMetadata& TExportToS3Response::Metadata() const { - return Metadata_; -} - -//////////////////////////////////////////////////////////////////////////////// - -class TExportClient::TImpl : public TClientImplCommon<TExportClient::TImpl> { -public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) +namespace NYdb { +namespace NExport { + +using namespace NThreading; +using namespace Ydb::Export; + +/// Common +namespace { + +TInstant ProtoTimestampToInstant(const NProtoBuf::Timestamp& timestamp) { + ui64 us = timestamp.seconds() * 1000000; + us += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(us); +} + +TVector<TExportItemProgress> ItemsProgressFromProto(const google::protobuf::RepeatedPtrField<ExportItemProgress>& proto) { + TVector<TExportItemProgress> result(Reserve(proto.size())); + + for (const auto& protoItem : proto) { + auto& item = result.emplace_back(); + item.PartsTotal = protoItem.parts_total(); + item.PartsCompleted = protoItem.parts_completed(); + item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); + item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); + } + + return result; +} + +} // anonymous + +/// YT +TExportToYtResponse::TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ExportToYtMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Host(metadata.settings().host()); + Metadata_.Settings.Port(metadata.settings().port()); + Metadata_.Settings.Token(metadata.settings().token()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + Metadata_.Settings.UseTypeV3(metadata.settings().use_type_v3()); + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TExportToYtResponse::TMetadata& TExportToYtResponse::Metadata() const { + return Metadata_; +} + +/// S3 +TExportToS3Response::TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ExportToS3Metadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Endpoint(metadata.settings().endpoint()); + Metadata_.Settings.Scheme(TProtoAccessor::FromProto<ExportToS3Settings>(metadata.settings().scheme())); + Metadata_.Settings.StorageClass(TProtoAccessor::FromProto(metadata.settings().storage_class())); + Metadata_.Settings.Bucket(metadata.settings().bucket()); + Metadata_.Settings.AccessKey(metadata.settings().access_key()); + Metadata_.Settings.SecretKey(metadata.settings().secret_key()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_prefix()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TExportToS3Response::TMetadata& TExportToS3Response::Metadata() const { + return Metadata_; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TExportClient::TImpl : public TClientImplCommon<TExportClient::TImpl> { +public: + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) : TClientImplCommon(std::move(connections), settings) - { - } - - TFuture<TExportToYtResponse> ExportToYt(ExportToYtRequest&& request, + { + } + + TFuture<TExportToYtResponse> ExportToYt(ExportToYtRequest&& request, const TExportToYtSettings& settings) - { - return RunOperation<V1::ExportService, ExportToYtRequest, ExportToYtResponse, TExportToYtResponse>( - std::move(request), - &V1::ExportService::Stub::AsyncExportToYt, + { + return RunOperation<V1::ExportService, ExportToYtRequest, ExportToYtResponse, TExportToYtResponse>( + std::move(request), + &V1::ExportService::Stub::AsyncExportToYt, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); - } - - TFuture<TExportToS3Response> ExportToS3(ExportToS3Request&& request, + } + + TFuture<TExportToS3Response> ExportToS3(ExportToS3Request&& request, const TExportToS3Settings& settings) - { - return RunOperation<V1::ExportService, ExportToS3Request, ExportToS3Response, TExportToS3Response>( - std::move(request), - &V1::ExportService::Stub::AsyncExportToS3, + { + return RunOperation<V1::ExportService, ExportToS3Request, ExportToS3Response, TExportToS3Response>( + std::move(request), + &V1::ExportService::Stub::AsyncExportToS3, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); - } - -}; - -//////////////////////////////////////////////////////////////////////////////// - -TExportClient::TExportClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) -{ -} - -TFuture<TExportToYtResponse> TExportClient::ExportToYt(const TExportToYtSettings& settings) { - auto request = MakeOperationRequest<ExportToYtRequest>(settings); - - request.mutable_settings()->set_host(settings.Host_); - request.mutable_settings()->set_port(settings.Port_.GetOrElse(80)); - request.mutable_settings()->set_token(settings.Token_); - - for (const auto& item : settings.Item_) { - auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_path(item.Src); - protoItem.set_destination_path(item.Dst); - } - - if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.GetRef()); - } - - if (settings.NumberOfRetries_) { - request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); - } - - request.mutable_settings()->set_use_type_v3(settings.UseTypeV3_); - + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TExportClient::TExportClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +TFuture<TExportToYtResponse> TExportClient::ExportToYt(const TExportToYtSettings& settings) { + auto request = MakeOperationRequest<ExportToYtRequest>(settings); + + request.mutable_settings()->set_host(settings.Host_); + request.mutable_settings()->set_port(settings.Port_.GetOrElse(80)); + request.mutable_settings()->set_token(settings.Token_); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_path(item.Src); + protoItem.set_destination_path(item.Dst); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(settings.Description_.GetRef()); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); + } + + request.mutable_settings()->set_use_type_v3(settings.UseTypeV3_); + return Impl_->ExportToYt(std::move(request), settings); -} - -TFuture<TExportToS3Response> TExportClient::ExportToS3(const TExportToS3Settings& settings) { - auto request = MakeOperationRequest<ExportToS3Request>(settings); - - request.mutable_settings()->set_endpoint(settings.Endpoint_); - request.mutable_settings()->set_scheme(TProtoAccessor::GetProto<ExportToS3Settings>(settings.Scheme_)); - request.mutable_settings()->set_storage_class(TProtoAccessor::GetProto(settings.StorageClass_)); - request.mutable_settings()->set_bucket(settings.Bucket_); - request.mutable_settings()->set_access_key(settings.AccessKey_); - request.mutable_settings()->set_secret_key(settings.SecretKey_); - - for (const auto& item : settings.Item_) { - auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_path(item.Src); - protoItem.set_destination_prefix(item.Dst); - } - - if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.GetRef()); - } - - if (settings.NumberOfRetries_) { - request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); - } - +} + +TFuture<TExportToS3Response> TExportClient::ExportToS3(const TExportToS3Settings& settings) { + auto request = MakeOperationRequest<ExportToS3Request>(settings); + + request.mutable_settings()->set_endpoint(settings.Endpoint_); + request.mutable_settings()->set_scheme(TProtoAccessor::GetProto<ExportToS3Settings>(settings.Scheme_)); + request.mutable_settings()->set_storage_class(TProtoAccessor::GetProto(settings.StorageClass_)); + request.mutable_settings()->set_bucket(settings.Bucket_); + request.mutable_settings()->set_access_key(settings.AccessKey_); + request.mutable_settings()->set_secret_key(settings.SecretKey_); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_path(item.Src); + protoItem.set_destination_prefix(item.Dst); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(settings.Description_.GetRef()); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); + } + return Impl_->ExportToS3(std::move(request), settings); -} - -} // namespace NExport -} // namespace NYdb +} + +} // namespace NExport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.h b/ydb/public/sdk/cpp/client/ydb_export/export.h index fdd250f7277..38225e36aa4 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.h +++ b/ydb/public/sdk/cpp/client/ydb_export/export.h @@ -1,136 +1,136 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> - + #include <ydb/public/sdk/cpp/client/ydb_types/s3_settings.h> - -namespace NYdb { -namespace NExport { - -/// Common -enum class EExportProgress { - Unspecified = 0, - Preparing = 1, - TransferData = 2, - Done = 3, - Cancellation = 4, - Cancelled = 5, - - Unknown = Max<int>(), -}; - -struct TExportItemProgress { - ui32 PartsTotal; - ui32 PartsCompleted; - TInstant StartTime; - TInstant EndTime; -}; - -/// YT -struct TExportToYtSettings : public TOperationRequestSettings<TExportToYtSettings> { - struct TItem { - TString Src; - TString Dst; - }; - - FLUENT_SETTING(TString, Host); - FLUENT_SETTING_OPTIONAL(ui16, Port); - FLUENT_SETTING(TString, Token); - FLUENT_SETTING_VECTOR(TItem, Item); - FLUENT_SETTING_OPTIONAL(TString, Description); - FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); - FLUENT_SETTING_DEFAULT(bool, UseTypeV3, false); -}; - -class TExportToYtResponse : public TOperation { -public: - struct TMetadata { - TExportToYtSettings Settings; - EExportProgress Progress; - TVector<TExportItemProgress> ItemsProgress; - }; - -public: - using TOperation::TOperation; - TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation); - - const TMetadata& Metadata() const; - -private: - TMetadata Metadata_; -}; - -/// S3 -struct TExportToS3Settings : public TOperationRequestSettings<TExportToS3Settings>, - public TS3Settings<TExportToS3Settings> { - using TSelf = TExportToS3Settings; - - enum class EStorageClass { - NOT_SET = 0, - STANDARD = 1, - REDUCED_REDUNDANCY = 2, - STANDARD_IA = 3, - ONEZONE_IA = 4, - INTELLIGENT_TIERING = 5, - GLACIER = 6, - DEEP_ARCHIVE = 7, - OUTPOSTS = 8, - - UNKNOWN = Max<int>(), - }; - - struct TItem { - TString Src; - TString Dst; - }; - - FLUENT_SETTING_DEFAULT(EStorageClass, StorageClass, EStorageClass::NOT_SET); - FLUENT_SETTING_VECTOR(TItem, Item); - FLUENT_SETTING_OPTIONAL(TString, Description); - FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); -}; - -class TExportToS3Response : public TOperation { -public: - struct TMetadata { - TExportToS3Settings Settings; - EExportProgress Progress; - TVector<TExportItemProgress> ItemsProgress; - }; - -public: - using TOperation::TOperation; - TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); - - const TMetadata& Metadata() const; - -private: - TMetadata Metadata_; -}; - -class TExportClient { - class TImpl; - -public: - TExportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - - NThreading::TFuture<TExportToYtResponse> ExportToYt(const TExportToYtSettings& settings); - NThreading::TFuture<TExportToS3Response> ExportToS3(const TExportToS3Settings& settings); - -private: - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NExport -} // namespace NYdb - -template<> -inline void Out<NYdb::NExport::TExportToYtResponse>(IOutputStream& o, const NYdb::NExport::TExportToYtResponse& x) { - return x.Out(o); -} - -template<> -inline void Out<NYdb::NExport::TExportToS3Response>(IOutputStream& o, const NYdb::NExport::TExportToS3Response& x) { - return x.Out(o); -} + +namespace NYdb { +namespace NExport { + +/// Common +enum class EExportProgress { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + Done = 3, + Cancellation = 4, + Cancelled = 5, + + Unknown = Max<int>(), +}; + +struct TExportItemProgress { + ui32 PartsTotal; + ui32 PartsCompleted; + TInstant StartTime; + TInstant EndTime; +}; + +/// YT +struct TExportToYtSettings : public TOperationRequestSettings<TExportToYtSettings> { + struct TItem { + TString Src; + TString Dst; + }; + + FLUENT_SETTING(TString, Host); + FLUENT_SETTING_OPTIONAL(ui16, Port); + FLUENT_SETTING(TString, Token); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(TString, Description); + FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); + FLUENT_SETTING_DEFAULT(bool, UseTypeV3, false); +}; + +class TExportToYtResponse : public TOperation { +public: + struct TMetadata { + TExportToYtSettings Settings; + EExportProgress Progress; + TVector<TExportItemProgress> ItemsProgress; + }; + +public: + using TOperation::TOperation; + TExportToYtResponse(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +/// S3 +struct TExportToS3Settings : public TOperationRequestSettings<TExportToS3Settings>, + public TS3Settings<TExportToS3Settings> { + using TSelf = TExportToS3Settings; + + enum class EStorageClass { + NOT_SET = 0, + STANDARD = 1, + REDUCED_REDUNDANCY = 2, + STANDARD_IA = 3, + ONEZONE_IA = 4, + INTELLIGENT_TIERING = 5, + GLACIER = 6, + DEEP_ARCHIVE = 7, + OUTPOSTS = 8, + + UNKNOWN = Max<int>(), + }; + + struct TItem { + TString Src; + TString Dst; + }; + + FLUENT_SETTING_DEFAULT(EStorageClass, StorageClass, EStorageClass::NOT_SET); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(TString, Description); + FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); +}; + +class TExportToS3Response : public TOperation { +public: + struct TMetadata { + TExportToS3Settings Settings; + EExportProgress Progress; + TVector<TExportItemProgress> ItemsProgress; + }; + +public: + using TOperation::TOperation; + TExportToS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +class TExportClient { + class TImpl; + +public: + TExportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + NThreading::TFuture<TExportToYtResponse> ExportToYt(const TExportToYtSettings& settings); + NThreading::TFuture<TExportToS3Response> ExportToS3(const TExportToS3Settings& settings); + +private: + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NExport +} // namespace NYdb + +template<> +inline void Out<NYdb::NExport::TExportToYtResponse>(IOutputStream& o, const NYdb::NExport::TExportToYtResponse& x) { + return x.Out(o); +} + +template<> +inline void Out<NYdb::NExport::TExportToS3Response>(IOutputStream& o, const NYdb::NExport::TExportToS3Response& x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp index e313c30e9ca..f0a55678efa 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp +++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp @@ -1,5 +1,5 @@ #include "import.h" - + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #undef INCLUDE_YDB_INTERNAL_H @@ -9,168 +9,168 @@ #include <ydb/public/api/protos/ydb_import.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - -namespace NYdb { -namespace NImport { - -using namespace NThreading; -using namespace Ydb::Import; - -/// Common -namespace { - -TInstant ProtoTimestampToInstant(const NProtoBuf::Timestamp& timestamp) { - ui64 us = timestamp.seconds() * 1000000; - us += timestamp.nanos() / 1000; - return TInstant::MicroSeconds(us); -} - -TVector<TImportItemProgress> ItemsProgressFromProto(const google::protobuf::RepeatedPtrField<Ydb::Import::ImportItemProgress>& proto) { - TVector<TImportItemProgress> result(Reserve(proto.size())); - - for (const auto& protoItem : proto) { - auto& item = result.emplace_back(); - item.PartsTotal = protoItem.parts_total(); - item.PartsCompleted = protoItem.parts_completed(); - item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); - item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); - } - - return result; -} - -} // anonymous - -/// S3 -TImportFromS3Response::TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) - : TOperation(std::move(status), std::move(operation)) -{ - ImportFromS3Metadata metadata; - GetProto().metadata().UnpackTo(&metadata); - - // settings - Metadata_.Settings.Endpoint(metadata.settings().endpoint()); - Metadata_.Settings.Scheme(TProtoAccessor::FromProto<ImportFromS3Settings>(metadata.settings().scheme())); - Metadata_.Settings.Bucket(metadata.settings().bucket()); - Metadata_.Settings.AccessKey(metadata.settings().access_key()); - Metadata_.Settings.SecretKey(metadata.settings().secret_key()); - - for (const auto& item : metadata.settings().items()) { - Metadata_.Settings.AppendItem({item.source_prefix(), item.destination_path()}); - } - - Metadata_.Settings.Description(metadata.settings().description()); - Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); - - // progress - Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); - Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); -} - -const TImportFromS3Response::TMetadata& TImportFromS3Response::Metadata() const { - return Metadata_; -} - -/// Data -TImportDataResult::TImportDataResult(TStatus&& status) - : TStatus(std::move(status)) -{} - -//////////////////////////////////////////////////////////////////////////////// - -class TImportClient::TImpl : public TClientImplCommon<TImportClient::TImpl> { -public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) - { - } - - TFuture<TImportFromS3Response> ImportFromS3(ImportFromS3Request&& request, const TImportFromS3Settings& settings) { - return RunOperation<V1::ImportService, ImportFromS3Request, ImportFromS3Response, TImportFromS3Response>( - std::move(request), - &V1::ImportService::Stub::AsyncImportFromS3, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - } - - template <typename TSettings> - TAsyncImportDataResult ImportData(ImportDataRequest&& request, const TSettings& settings) { - auto promise = NThreading::NewPromise<TImportDataResult>(); - + +namespace NYdb { +namespace NImport { + +using namespace NThreading; +using namespace Ydb::Import; + +/// Common +namespace { + +TInstant ProtoTimestampToInstant(const NProtoBuf::Timestamp& timestamp) { + ui64 us = timestamp.seconds() * 1000000; + us += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(us); +} + +TVector<TImportItemProgress> ItemsProgressFromProto(const google::protobuf::RepeatedPtrField<Ydb::Import::ImportItemProgress>& proto) { + TVector<TImportItemProgress> result(Reserve(proto.size())); + + for (const auto& protoItem : proto) { + auto& item = result.emplace_back(); + item.PartsTotal = protoItem.parts_total(); + item.PartsCompleted = protoItem.parts_completed(); + item.StartTime = ProtoTimestampToInstant(protoItem.start_time()); + item.EndTime = ProtoTimestampToInstant(protoItem.end_time()); + } + + return result; +} + +} // anonymous + +/// S3 +TImportFromS3Response::TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ImportFromS3Metadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.Endpoint(metadata.settings().endpoint()); + Metadata_.Settings.Scheme(TProtoAccessor::FromProto<ImportFromS3Settings>(metadata.settings().scheme())); + Metadata_.Settings.Bucket(metadata.settings().bucket()); + Metadata_.Settings.AccessKey(metadata.settings().access_key()); + Metadata_.Settings.SecretKey(metadata.settings().secret_key()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_prefix(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TImportFromS3Response::TMetadata& TImportFromS3Response::Metadata() const { + return Metadata_; +} + +/// Data +TImportDataResult::TImportDataResult(TStatus&& status) + : TStatus(std::move(status)) +{} + +//////////////////////////////////////////////////////////////////////////////// + +class TImportClient::TImpl : public TClientImplCommon<TImportClient::TImpl> { +public: + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TFuture<TImportFromS3Response> ImportFromS3(ImportFromS3Request&& request, const TImportFromS3Settings& settings) { + return RunOperation<V1::ImportService, ImportFromS3Request, ImportFromS3Response, TImportFromS3Response>( + std::move(request), + &V1::ImportService::Stub::AsyncImportFromS3, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + } + + template <typename TSettings> + TAsyncImportDataResult ImportData(ImportDataRequest&& request, const TSettings& settings) { + auto promise = NThreading::NewPromise<TImportDataResult>(); + auto extractor = [promise] - (google::protobuf::Any*, TPlainStatus status) mutable { + (google::protobuf::Any*, TPlainStatus status) mutable { TImportDataResult result(TStatus(std::move(status))); - promise.SetValue(std::move(result)); - }; - - Connections_->RunDeferred<V1::ImportService, ImportDataRequest, ImportDataResponse>( - std::move(request), - extractor, - &V1::ImportService::Stub::AsyncImportData, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return promise.GetFuture(); - } - - template <typename TData> - TAsyncImportDataResult ImportData(const TString& table, TData&& data, const TImportYdbDumpDataSettings& settings) { - auto request = MakeOperationRequest<ImportDataRequest>(settings); - - request.set_path(table); - request.set_data(std::forward<TData>(data)); - - for (const auto& column : settings.Columns_) { - request.mutable_ydb_dump()->add_columns(column); - } - - return ImportData(std::move(request), settings); - } - -}; - -//////////////////////////////////////////////////////////////////////////////// - -TImportClient::TImportClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) -{ -} - -TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { - auto request = MakeOperationRequest<ImportFromS3Request>(settings); - - request.mutable_settings()->set_endpoint(settings.Endpoint_); - request.mutable_settings()->set_scheme(TProtoAccessor::GetProto<ImportFromS3Settings>(settings.Scheme_)); - request.mutable_settings()->set_bucket(settings.Bucket_); - request.mutable_settings()->set_access_key(settings.AccessKey_); - request.mutable_settings()->set_secret_key(settings.SecretKey_); - - for (const auto& item : settings.Item_) { - auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_prefix(item.Src); - protoItem.set_destination_path(item.Dst); - } - - if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.GetRef()); - } - - if (settings.NumberOfRetries_) { - request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); - } - - return Impl_->ImportFromS3(std::move(request), settings); -} - -TAsyncImportDataResult TImportClient::ImportData(const TString& table, TString&& data, const TImportYdbDumpDataSettings& settings) { - return Impl_->ImportData(table, std::move(data), settings); -} - -TAsyncImportDataResult TImportClient::ImportData(const TString& table, const TString& data, const TImportYdbDumpDataSettings& settings) { - return Impl_->ImportData(table, data, settings); -} - -} // namespace NImport -} // namespace NYdb + promise.SetValue(std::move(result)); + }; + + Connections_->RunDeferred<V1::ImportService, ImportDataRequest, ImportDataResponse>( + std::move(request), + extractor, + &V1::ImportService::Stub::AsyncImportData, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return promise.GetFuture(); + } + + template <typename TData> + TAsyncImportDataResult ImportData(const TString& table, TData&& data, const TImportYdbDumpDataSettings& settings) { + auto request = MakeOperationRequest<ImportDataRequest>(settings); + + request.set_path(table); + request.set_data(std::forward<TData>(data)); + + for (const auto& column : settings.Columns_) { + request.mutable_ydb_dump()->add_columns(column); + } + + return ImportData(std::move(request), settings); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TImportClient::TImportClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { + auto request = MakeOperationRequest<ImportFromS3Request>(settings); + + request.mutable_settings()->set_endpoint(settings.Endpoint_); + request.mutable_settings()->set_scheme(TProtoAccessor::GetProto<ImportFromS3Settings>(settings.Scheme_)); + request.mutable_settings()->set_bucket(settings.Bucket_); + request.mutable_settings()->set_access_key(settings.AccessKey_); + request.mutable_settings()->set_secret_key(settings.SecretKey_); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_prefix(item.Src); + protoItem.set_destination_path(item.Dst); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(settings.Description_.GetRef()); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); + } + + return Impl_->ImportFromS3(std::move(request), settings); +} + +TAsyncImportDataResult TImportClient::ImportData(const TString& table, TString&& data, const TImportYdbDumpDataSettings& settings) { + return Impl_->ImportData(table, std::move(data), settings); +} + +TAsyncImportDataResult TImportClient::ImportData(const TString& table, const TString& data, const TImportYdbDumpDataSettings& settings) { + return Impl_->ImportData(table, data, settings); +} + +} // namespace NImport +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h index 4e352b16b3d..865fec384c6 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.h +++ b/ydb/public/sdk/cpp/client/ydb_import/import.h @@ -1,102 +1,102 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> - + #include <ydb/public/sdk/cpp/client/ydb_types/s3_settings.h> - -namespace NYdb { -namespace NImport { - -/// Common -enum class EImportProgress { - Unspecified = 0, - Preparing = 1, - TransferData = 2, - BuildIndexes = 3, - Done = 4, - Cancellation = 5, - Cancelled = 6, - - Unknown = Max<int>(), -}; - -struct TImportItemProgress { - ui32 PartsTotal; - ui32 PartsCompleted; - TInstant StartTime; - TInstant EndTime; -}; - -/// S3 -struct TImportFromS3Settings : public TOperationRequestSettings<TImportFromS3Settings>, - public TS3Settings<TImportFromS3Settings> { - using TSelf = TImportFromS3Settings; - - struct TItem { - TString Src; - TString Dst; - }; - - FLUENT_SETTING_VECTOR(TItem, Item); - FLUENT_SETTING_OPTIONAL(TString, Description); - FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); -}; - -class TImportFromS3Response : public TOperation { -public: - struct TMetadata { - TImportFromS3Settings Settings; - EImportProgress Progress; - TVector<TImportItemProgress> ItemsProgress; - }; - -public: - using TOperation::TOperation; - TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); - - const TMetadata& Metadata() const; - -private: - TMetadata Metadata_; -}; - -/// Data -struct TImportYdbDumpDataSettings : public TOperationRequestSettings<TImportYdbDumpDataSettings> { - using TSelf = TImportYdbDumpDataSettings; - - FLUENT_SETTING_VECTOR(TString, Columns); - - using TOperationRequestSettings::TOperationRequestSettings; -}; - -class TImportDataResult : public TStatus { -public: - explicit TImportDataResult(TStatus&& status); -}; - -using TAsyncImportDataResult = NThreading::TFuture<TImportDataResult>; - -class TImportClient { - class TImpl; - -public: - TImportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - - NThreading::TFuture<TImportFromS3Response> ImportFromS3(const TImportFromS3Settings& settings); - - // ydb dump format - TAsyncImportDataResult ImportData(const TString& table, TString&& data, const TImportYdbDumpDataSettings& settings); - TAsyncImportDataResult ImportData(const TString& table, const TString& data, const TImportYdbDumpDataSettings& settings); - -private: - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NImport -} // namespace NYdb - -template<> -inline void Out<NYdb::NImport::TImportFromS3Response>(IOutputStream& o, const NYdb::NImport::TImportFromS3Response& x) { - return x.Out(o); -} + +namespace NYdb { +namespace NImport { + +/// Common +enum class EImportProgress { + Unspecified = 0, + Preparing = 1, + TransferData = 2, + BuildIndexes = 3, + Done = 4, + Cancellation = 5, + Cancelled = 6, + + Unknown = Max<int>(), +}; + +struct TImportItemProgress { + ui32 PartsTotal; + ui32 PartsCompleted; + TInstant StartTime; + TInstant EndTime; +}; + +/// S3 +struct TImportFromS3Settings : public TOperationRequestSettings<TImportFromS3Settings>, + public TS3Settings<TImportFromS3Settings> { + using TSelf = TImportFromS3Settings; + + struct TItem { + TString Src; + TString Dst; + }; + + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(TString, Description); + FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); +}; + +class TImportFromS3Response : public TOperation { +public: + struct TMetadata { + TImportFromS3Settings Settings; + EImportProgress Progress; + TVector<TImportItemProgress> ItemsProgress; + }; + +public: + using TOperation::TOperation; + TImportFromS3Response(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +/// Data +struct TImportYdbDumpDataSettings : public TOperationRequestSettings<TImportYdbDumpDataSettings> { + using TSelf = TImportYdbDumpDataSettings; + + FLUENT_SETTING_VECTOR(TString, Columns); + + using TOperationRequestSettings::TOperationRequestSettings; +}; + +class TImportDataResult : public TStatus { +public: + explicit TImportDataResult(TStatus&& status); +}; + +using TAsyncImportDataResult = NThreading::TFuture<TImportDataResult>; + +class TImportClient { + class TImpl; + +public: + TImportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + NThreading::TFuture<TImportFromS3Response> ImportFromS3(const TImportFromS3Settings& settings); + + // ydb dump format + TAsyncImportDataResult ImportData(const TString& table, TString&& data, const TImportYdbDumpDataSettings& settings); + TAsyncImportDataResult ImportData(const TString& table, const TString& data, const TImportYdbDumpDataSettings& settings); + +private: + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NImport +} // namespace NYdb + +template<> +inline void Out<NYdb::NImport::TImportFromS3Response>(IOutputStream& o, const NYdb::NImport::TImportFromS3Response& x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp index ae26ee43d4e..fac441c5050 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp @@ -1,193 +1,193 @@ #include "operation.h" - + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #undef INCLUDE_YDB_INTERNAL_H -/* Headers below used to instantiate concrete 'Get' & 'List' methods */ +/* Headers below used to instantiate concrete 'Get' & 'List' methods */ #include <ydb/public/sdk/cpp/client/ydb_export/export.h> #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <ydb/public/lib/operation_id/operation_id.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - -namespace NYdb { -namespace NOperation { - + +namespace NYdb { +namespace NOperation { + constexpr TDuration OPERATION_CLIENT_TIMEOUT = TDuration::Seconds(5); -using namespace NThreading; -using namespace Ydb::Operation; -using namespace Ydb::Operations; - -class TOperationClient::TImpl : public TClientImplCommon<TOperationClient::TImpl> { - template <typename TRequest, typename TResponse> - using TSimpleRpc = TGRpcConnectionsImpl::TSimpleRpc<V1::OperationService, TRequest, TResponse>; - - template <typename TRequest, typename TResponse> - TAsyncStatus Run(TRequest&& request, TSimpleRpc<TRequest, TResponse> rpc) { - auto promise = NewPromise<TStatus>(); - +using namespace NThreading; +using namespace Ydb::Operation; +using namespace Ydb::Operations; + +class TOperationClient::TImpl : public TClientImplCommon<TOperationClient::TImpl> { + template <typename TRequest, typename TResponse> + using TSimpleRpc = TGRpcConnectionsImpl::TSimpleRpc<V1::OperationService, TRequest, TResponse>; + + template <typename TRequest, typename TResponse> + TAsyncStatus Run(TRequest&& request, TSimpleRpc<TRequest, TResponse> rpc) { + auto promise = NewPromise<TStatus>(); + auto extractor = [promise] (TResponse* response, TPlainStatus status) mutable { - if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + if (response) { + NYql::TIssues opIssues; + NYql::IssuesFromMessage(response->issues(), opIssues); TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); - promise.SetValue(std::move(st)); - } else { + promise.SetValue(std::move(st)); + } else { TStatus st(std::move(status)); - promise.SetValue(std::move(st)); - } - }; - - Connections_->Run<V1::OperationService, TRequest, TResponse>( - std::move(request), - extractor, - rpc, + promise.SetValue(std::move(st)); + } + }; + + Connections_->Run<V1::OperationService, TRequest, TResponse>( + std::move(request), + extractor, + rpc, DbDriverState_, TRpcRequestSettings(), OPERATION_CLIENT_TIMEOUT, TString()); - - return promise.GetFuture(); - } - -public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) + + return promise.GetFuture(); + } + +public: + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) : TClientImplCommon(std::move(connections), settings) - { - } - - template <typename TOp> - TFuture<TOp> Get(GetOperationRequest&& request) { - return RunOperation<V1::OperationService, GetOperationRequest, GetOperationResponse, TOp>( - std::move(request), - &V1::OperationService::Stub::AsyncGetOperation); - } - - TAsyncStatus Cancel(CancelOperationRequest&& request) { - return Run<CancelOperationRequest, CancelOperationResponse>(std::move(request), - &V1::OperationService::Stub::AsyncCancelOperation); - } - - TAsyncStatus Forget(ForgetOperationRequest&& request) { - return Run<ForgetOperationRequest, ForgetOperationResponse>(std::move(request), - &V1::OperationService::Stub::AsyncForgetOperation); - } - - template <typename TOp> - TFuture<TOperationsList<TOp>> List(ListOperationsRequest&& request) { - auto promise = NewPromise<TOperationsList<TOp>>(); - + { + } + + template <typename TOp> + TFuture<TOp> Get(GetOperationRequest&& request) { + return RunOperation<V1::OperationService, GetOperationRequest, GetOperationResponse, TOp>( + std::move(request), + &V1::OperationService::Stub::AsyncGetOperation); + } + + TAsyncStatus Cancel(CancelOperationRequest&& request) { + return Run<CancelOperationRequest, CancelOperationResponse>(std::move(request), + &V1::OperationService::Stub::AsyncCancelOperation); + } + + TAsyncStatus Forget(ForgetOperationRequest&& request) { + return Run<ForgetOperationRequest, ForgetOperationResponse>(std::move(request), + &V1::OperationService::Stub::AsyncForgetOperation); + } + + template <typename TOp> + TFuture<TOperationsList<TOp>> List(ListOperationsRequest&& request) { + auto promise = NewPromise<TOperationsList<TOp>>(); + auto extractor = [promise] (ListOperationsResponse* response, TPlainStatus status) mutable { - if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + if (response) { + NYql::TIssues opIssues; + NYql::IssuesFromMessage(response->issues(), opIssues); TStatus st(static_cast<EStatus>(response->status()), std::move(opIssues)); - - TVector<TOp> operations(Reserve(response->operations_size())); - for (auto& operation : *response->mutable_operations()) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(operation.issues(), opIssues); - operations.emplace_back( + + TVector<TOp> operations(Reserve(response->operations_size())); + for (auto& operation : *response->mutable_operations()) { + NYql::TIssues opIssues; + NYql::IssuesFromMessage(operation.issues(), opIssues); + operations.emplace_back( TStatus(static_cast<EStatus>(operation.status()), std::move(opIssues)), - std::move(operation)); - } - - promise.SetValue(TOperationsList<TOp>(std::move(st), std::move(operations), response->next_page_token())); - } else { + std::move(operation)); + } + + promise.SetValue(TOperationsList<TOp>(std::move(st), std::move(operations), response->next_page_token())); + } else { TStatus st(std::move(status)); - promise.SetValue(TOperationsList<TOp>(std::move(st))); - } - }; - - Connections_->Run<V1::OperationService, ListOperationsRequest, ListOperationsResponse>( - std::move(request), - extractor, - &V1::OperationService::Stub::AsyncListOperations, + promise.SetValue(TOperationsList<TOp>(std::move(st))); + } + }; + + Connections_->Run<V1::OperationService, ListOperationsRequest, ListOperationsResponse>( + std::move(request), + extractor, + &V1::OperationService::Stub::AsyncListOperations, DbDriverState_, TRpcRequestSettings(), OPERATION_CLIENT_TIMEOUT, TString()); - - return promise.GetFuture(); - } - -}; - -//////////////////////////////////////////////////////////////////////////////// - -TOperationClient::TOperationClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TImpl(CreateInternalInterface(driver), settings)) -{ -} - -template <typename TOp> -TFuture<TOp> TOperationClient::Get(const TOperation::TOperationId& id) { - auto request = MakeRequest<GetOperationRequest>(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); - - return Impl_->Get<TOp>(std::move(request)); -} - -TAsyncStatus TOperationClient::Cancel(const TOperation::TOperationId& id) { - auto request = MakeRequest<CancelOperationRequest>(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); - - return Impl_->Cancel(std::move(request)); -} - -TAsyncStatus TOperationClient::Forget(const TOperation::TOperationId& id) { - auto request = MakeRequest<ForgetOperationRequest>(); - request.set_id(NKikimr::NOperationId::ProtoToString(id)); - - return Impl_->Forget(std::move(request)); -} - -template <typename TOp> -TFuture<TOperationsList<TOp>> TOperationClient::List(const TString& kind, ui64 pageSize, const TString& pageToken) { - auto request = MakeRequest<ListOperationsRequest>(); - - request.set_kind(kind); - if (pageSize) { - request.set_page_size(pageSize); - } - if (pageToken) { - request.set_page_token(pageToken); - } - - return Impl_->List<TOp>(std::move(request)); -} - -// Instantiations -template TFuture<NExport::TExportToYtResponse> TOperationClient::Get(const TOperation::TOperationId& id); -template <> -TFuture<TOperationsList<NExport::TExportToYtResponse>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { - // TODO: export -> export/yt - return List<NExport::TExportToYtResponse>("export", pageSize, pageToken); -} - -template TFuture<NExport::TExportToS3Response> TOperationClient::Get(const TOperation::TOperationId& id); -template <> -TFuture<TOperationsList<NExport::TExportToS3Response>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { - return List<NExport::TExportToS3Response>("export/s3", pageSize, pageToken); -} - -template TFuture<NImport::TImportFromS3Response> TOperationClient::Get(const TOperation::TOperationId& id); -template <> -TFuture<TOperationsList<NImport::TImportFromS3Response>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { - return List<NImport::TImportFromS3Response>("import/s3", pageSize, pageToken); -} - + + return promise.GetFuture(); + } + +}; + +//////////////////////////////////////////////////////////////////////////////// + +TOperationClient::TOperationClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{ +} + +template <typename TOp> +TFuture<TOp> TOperationClient::Get(const TOperation::TOperationId& id) { + auto request = MakeRequest<GetOperationRequest>(); + request.set_id(NKikimr::NOperationId::ProtoToString(id)); + + return Impl_->Get<TOp>(std::move(request)); +} + +TAsyncStatus TOperationClient::Cancel(const TOperation::TOperationId& id) { + auto request = MakeRequest<CancelOperationRequest>(); + request.set_id(NKikimr::NOperationId::ProtoToString(id)); + + return Impl_->Cancel(std::move(request)); +} + +TAsyncStatus TOperationClient::Forget(const TOperation::TOperationId& id) { + auto request = MakeRequest<ForgetOperationRequest>(); + request.set_id(NKikimr::NOperationId::ProtoToString(id)); + + return Impl_->Forget(std::move(request)); +} + +template <typename TOp> +TFuture<TOperationsList<TOp>> TOperationClient::List(const TString& kind, ui64 pageSize, const TString& pageToken) { + auto request = MakeRequest<ListOperationsRequest>(); + + request.set_kind(kind); + if (pageSize) { + request.set_page_size(pageSize); + } + if (pageToken) { + request.set_page_token(pageToken); + } + + return Impl_->List<TOp>(std::move(request)); +} + +// Instantiations +template TFuture<NExport::TExportToYtResponse> TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture<TOperationsList<NExport::TExportToYtResponse>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { + // TODO: export -> export/yt + return List<NExport::TExportToYtResponse>("export", pageSize, pageToken); +} + +template TFuture<NExport::TExportToS3Response> TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture<TOperationsList<NExport::TExportToS3Response>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { + return List<NExport::TExportToS3Response>("export/s3", pageSize, pageToken); +} + +template TFuture<NImport::TImportFromS3Response> TOperationClient::Get(const TOperation::TOperationId& id); +template <> +TFuture<TOperationsList<NImport::TImportFromS3Response>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { + return List<NImport::TImportFromS3Response>("import/s3", pageSize, pageToken); +} + template TFuture<NTable::TBuildIndexOperation> TOperationClient::Get(const TOperation::TOperationId& id); template <> TFuture<TOperationsList<NTable::TBuildIndexOperation>> TOperationClient::List(ui64 pageSize, const TString& pageToken) { return List<NTable::TBuildIndexOperation>("buildindex", pageSize, pageToken); } -} // namespace NOperation -} // namespace NYdb +} // namespace NOperation +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.h b/ydb/public/sdk/cpp/client/ydb_operation/operation.h index d1c14015c84..794741331fc 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.h @@ -1,72 +1,72 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> - -#include <util/string/builder.h> - -namespace NYdb { -namespace NOperation { - -template <typename TOp> -class TOperationsList : public TStatus { -public: - TOperationsList(TStatus&& status) - : TStatus(std::move(status)) - { - } - - TOperationsList(TStatus&& status, TVector<TOp>&& operations, const TString& nextPageToken) - : TStatus(std::move(status)) - , Operations_(std::move(operations)) - , NextPageToken_(nextPageToken) - { - } - - const TVector<TOp>& GetList() const { return Operations_; } - const TString& NextPageToken() const { return NextPageToken_; } - - TString ToJsonString() const { - TStringBuilder json; - json << "{\"operations\":["; - - bool first = true; - for (const auto& operation : GetList()) { - json << (first ? "" : ",") << operation.ToJsonString(); - first = false; - } - - json << "],\"nextPageToken\":\"" << NextPageToken() << "\"}"; - return json; - } - -private: - TVector<TOp> Operations_; - TString NextPageToken_; -}; - -class TOperationClient { - class TImpl; - -public: - TOperationClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - - template <typename TOp> - NThreading::TFuture<TOp> Get(const TOperation::TOperationId& id); - - TAsyncStatus Cancel(const TOperation::TOperationId& id); - TAsyncStatus Forget(const TOperation::TOperationId& id); - - template <typename TOp> - NThreading::TFuture<TOperationsList<TOp>> List(ui64 pageSize = 0, const TString& pageToken = TString()); - -private: - template <typename TOp> - NThreading::TFuture<TOperationsList<TOp>> List(const TString& kind, ui64 pageSize, const TString& pageToken); - -private: - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NOperation -} // namespace NYdb + +#include <util/string/builder.h> + +namespace NYdb { +namespace NOperation { + +template <typename TOp> +class TOperationsList : public TStatus { +public: + TOperationsList(TStatus&& status) + : TStatus(std::move(status)) + { + } + + TOperationsList(TStatus&& status, TVector<TOp>&& operations, const TString& nextPageToken) + : TStatus(std::move(status)) + , Operations_(std::move(operations)) + , NextPageToken_(nextPageToken) + { + } + + const TVector<TOp>& GetList() const { return Operations_; } + const TString& NextPageToken() const { return NextPageToken_; } + + TString ToJsonString() const { + TStringBuilder json; + json << "{\"operations\":["; + + bool first = true; + for (const auto& operation : GetList()) { + json << (first ? "" : ",") << operation.ToJsonString(); + first = false; + } + + json << "],\"nextPageToken\":\"" << NextPageToken() << "\"}"; + return json; + } + +private: + TVector<TOp> Operations_; + TString NextPageToken_; +}; + +class TOperationClient { + class TImpl; + +public: + TOperationClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + template <typename TOp> + NThreading::TFuture<TOp> Get(const TOperation::TOperationId& id); + + TAsyncStatus Cancel(const TOperation::TOperationId& id); + TAsyncStatus Forget(const TOperation::TOperationId& id); + + template <typename TOp> + NThreading::TFuture<TOperationsList<TOp>> List(ui64 pageSize = 0, const TString& pageToken = TString()); + +private: + template <typename TOp> + NThreading::TFuture<TOperationsList<TOp>> List(const TString& kind, ui64 pageSize, const TString& pageToken); + +private: + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NOperation +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index 574d84b7ae9..dd02fb89dbd 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -12,119 +12,119 @@ const Ydb::Value& TProtoAccessor::GetProto(const TValue& value) { return value.GetProto(); } -// exports & imports -template <typename TProtoSettings> -typename TProtoSettings::Scheme TProtoAccessor::GetProto(ES3Scheme value) { - switch (value) { - case ES3Scheme::HTTP: - return TProtoSettings::HTTP; - case ES3Scheme::HTTPS: - return TProtoSettings::HTTPS; - } -} - -template Ydb::Export::ExportToS3Settings::Scheme TProtoAccessor::GetProto<Ydb::Export::ExportToS3Settings>(ES3Scheme value); -template Ydb::Import::ImportFromS3Settings::Scheme TProtoAccessor::GetProto<Ydb::Import::ImportFromS3Settings>(ES3Scheme value); - -template <typename TProtoSettings> -ES3Scheme TProtoAccessor::FromProto(typename TProtoSettings::Scheme value) { - switch (value) { - case TProtoSettings::HTTP: - return ES3Scheme::HTTP; - default: - return ES3Scheme::HTTPS; - } -} - -template ES3Scheme TProtoAccessor::FromProto<Ydb::Export::ExportToS3Settings>(Ydb::Export::ExportToS3Settings::Scheme value); -template ES3Scheme TProtoAccessor::FromProto<Ydb::Import::ImportFromS3Settings>(Ydb::Import::ImportFromS3Settings::Scheme value); - -Ydb::Export::ExportToS3Settings::StorageClass TProtoAccessor::GetProto(NExport::TExportToS3Settings::EStorageClass value) { - switch (value) { - case NExport::TExportToS3Settings::EStorageClass::STANDARD: - return Ydb::Export::ExportToS3Settings::STANDARD; - case NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY: - return Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY; - case NExport::TExportToS3Settings::EStorageClass::STANDARD_IA: - return Ydb::Export::ExportToS3Settings::STANDARD_IA; - case NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA: - return Ydb::Export::ExportToS3Settings::ONEZONE_IA; - case NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING: - return Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING; - case NExport::TExportToS3Settings::EStorageClass::GLACIER: - return Ydb::Export::ExportToS3Settings::GLACIER; - case NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE: - return Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE; - case NExport::TExportToS3Settings::EStorageClass::OUTPOSTS: - return Ydb::Export::ExportToS3Settings::OUTPOSTS; - default: - return Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED; - } -} - -NExport::TExportToS3Settings::EStorageClass TProtoAccessor::FromProto(Ydb::Export::ExportToS3Settings::StorageClass value) { - switch (value) { - case Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED: - return NExport::TExportToS3Settings::EStorageClass::NOT_SET; - case Ydb::Export::ExportToS3Settings::STANDARD: - return NExport::TExportToS3Settings::EStorageClass::STANDARD; - case Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY: - return NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY; - case Ydb::Export::ExportToS3Settings::STANDARD_IA: - return NExport::TExportToS3Settings::EStorageClass::STANDARD_IA; - case Ydb::Export::ExportToS3Settings::ONEZONE_IA: - return NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA; - case Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING: - return NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING; - case Ydb::Export::ExportToS3Settings::GLACIER: - return NExport::TExportToS3Settings::EStorageClass::GLACIER; - case Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE: - return NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE; - case Ydb::Export::ExportToS3Settings::OUTPOSTS: - return NExport::TExportToS3Settings::EStorageClass::OUTPOSTS; - default: - return NExport::TExportToS3Settings::EStorageClass::UNKNOWN; - } -} - -NExport::EExportProgress TProtoAccessor::FromProto(Ydb::Export::ExportProgress::Progress value) { - switch (value) { - case Ydb::Export::ExportProgress::PROGRESS_UNSPECIFIED: - return NExport::EExportProgress::Unspecified; - case Ydb::Export::ExportProgress::PROGRESS_PREPARING: - return NExport::EExportProgress::Preparing; - case Ydb::Export::ExportProgress::PROGRESS_TRANSFER_DATA: - return NExport::EExportProgress::TransferData; - case Ydb::Export::ExportProgress::PROGRESS_DONE: - return NExport::EExportProgress::Done; - case Ydb::Export::ExportProgress::PROGRESS_CANCELLATION: - return NExport::EExportProgress::Cancellation; - case Ydb::Export::ExportProgress::PROGRESS_CANCELLED: - return NExport::EExportProgress::Cancelled; - default: - return NExport::EExportProgress::Unknown; - } -} - -NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress::Progress value) { - switch (value) { - case Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED: - return NImport::EImportProgress::Unspecified; - case Ydb::Import::ImportProgress::PROGRESS_PREPARING: - return NImport::EImportProgress::Preparing; - case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: - return NImport::EImportProgress::TransferData; - case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: - return NImport::EImportProgress::BuildIndexes; - case Ydb::Import::ImportProgress::PROGRESS_DONE: - return NImport::EImportProgress::Done; - case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: - return NImport::EImportProgress::Cancellation; - case Ydb::Import::ImportProgress::PROGRESS_CANCELLED: - return NImport::EImportProgress::Cancelled; - default: - return NImport::EImportProgress::Unknown; - } -} - +// exports & imports +template <typename TProtoSettings> +typename TProtoSettings::Scheme TProtoAccessor::GetProto(ES3Scheme value) { + switch (value) { + case ES3Scheme::HTTP: + return TProtoSettings::HTTP; + case ES3Scheme::HTTPS: + return TProtoSettings::HTTPS; + } +} + +template Ydb::Export::ExportToS3Settings::Scheme TProtoAccessor::GetProto<Ydb::Export::ExportToS3Settings>(ES3Scheme value); +template Ydb::Import::ImportFromS3Settings::Scheme TProtoAccessor::GetProto<Ydb::Import::ImportFromS3Settings>(ES3Scheme value); + +template <typename TProtoSettings> +ES3Scheme TProtoAccessor::FromProto(typename TProtoSettings::Scheme value) { + switch (value) { + case TProtoSettings::HTTP: + return ES3Scheme::HTTP; + default: + return ES3Scheme::HTTPS; + } +} + +template ES3Scheme TProtoAccessor::FromProto<Ydb::Export::ExportToS3Settings>(Ydb::Export::ExportToS3Settings::Scheme value); +template ES3Scheme TProtoAccessor::FromProto<Ydb::Import::ImportFromS3Settings>(Ydb::Import::ImportFromS3Settings::Scheme value); + +Ydb::Export::ExportToS3Settings::StorageClass TProtoAccessor::GetProto(NExport::TExportToS3Settings::EStorageClass value) { + switch (value) { + case NExport::TExportToS3Settings::EStorageClass::STANDARD: + return Ydb::Export::ExportToS3Settings::STANDARD; + case NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY: + return Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY; + case NExport::TExportToS3Settings::EStorageClass::STANDARD_IA: + return Ydb::Export::ExportToS3Settings::STANDARD_IA; + case NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA: + return Ydb::Export::ExportToS3Settings::ONEZONE_IA; + case NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING: + return Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING; + case NExport::TExportToS3Settings::EStorageClass::GLACIER: + return Ydb::Export::ExportToS3Settings::GLACIER; + case NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE: + return Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE; + case NExport::TExportToS3Settings::EStorageClass::OUTPOSTS: + return Ydb::Export::ExportToS3Settings::OUTPOSTS; + default: + return Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED; + } +} + +NExport::TExportToS3Settings::EStorageClass TProtoAccessor::FromProto(Ydb::Export::ExportToS3Settings::StorageClass value) { + switch (value) { + case Ydb::Export::ExportToS3Settings::STORAGE_CLASS_UNSPECIFIED: + return NExport::TExportToS3Settings::EStorageClass::NOT_SET; + case Ydb::Export::ExportToS3Settings::STANDARD: + return NExport::TExportToS3Settings::EStorageClass::STANDARD; + case Ydb::Export::ExportToS3Settings::REDUCED_REDUNDANCY: + return NExport::TExportToS3Settings::EStorageClass::REDUCED_REDUNDANCY; + case Ydb::Export::ExportToS3Settings::STANDARD_IA: + return NExport::TExportToS3Settings::EStorageClass::STANDARD_IA; + case Ydb::Export::ExportToS3Settings::ONEZONE_IA: + return NExport::TExportToS3Settings::EStorageClass::ONEZONE_IA; + case Ydb::Export::ExportToS3Settings::INTELLIGENT_TIERING: + return NExport::TExportToS3Settings::EStorageClass::INTELLIGENT_TIERING; + case Ydb::Export::ExportToS3Settings::GLACIER: + return NExport::TExportToS3Settings::EStorageClass::GLACIER; + case Ydb::Export::ExportToS3Settings::DEEP_ARCHIVE: + return NExport::TExportToS3Settings::EStorageClass::DEEP_ARCHIVE; + case Ydb::Export::ExportToS3Settings::OUTPOSTS: + return NExport::TExportToS3Settings::EStorageClass::OUTPOSTS; + default: + return NExport::TExportToS3Settings::EStorageClass::UNKNOWN; + } +} + +NExport::EExportProgress TProtoAccessor::FromProto(Ydb::Export::ExportProgress::Progress value) { + switch (value) { + case Ydb::Export::ExportProgress::PROGRESS_UNSPECIFIED: + return NExport::EExportProgress::Unspecified; + case Ydb::Export::ExportProgress::PROGRESS_PREPARING: + return NExport::EExportProgress::Preparing; + case Ydb::Export::ExportProgress::PROGRESS_TRANSFER_DATA: + return NExport::EExportProgress::TransferData; + case Ydb::Export::ExportProgress::PROGRESS_DONE: + return NExport::EExportProgress::Done; + case Ydb::Export::ExportProgress::PROGRESS_CANCELLATION: + return NExport::EExportProgress::Cancellation; + case Ydb::Export::ExportProgress::PROGRESS_CANCELLED: + return NExport::EExportProgress::Cancelled; + default: + return NExport::EExportProgress::Unknown; + } +} + +NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress::Progress value) { + switch (value) { + case Ydb::Import::ImportProgress::PROGRESS_UNSPECIFIED: + return NImport::EImportProgress::Unspecified; + case Ydb::Import::ImportProgress::PROGRESS_PREPARING: + return NImport::EImportProgress::Preparing; + case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: + return NImport::EImportProgress::TransferData; + case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: + return NImport::EImportProgress::BuildIndexes; + case Ydb::Import::ImportProgress::PROGRESS_DONE: + return NImport::EImportProgress::Done; + case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: + return NImport::EImportProgress::Cancellation; + case Ydb::Import::ImportProgress::PROGRESS_CANCELLED: + return NImport::EImportProgress::Cancelled; + default: + return NImport::EImportProgress::Unknown; + } +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h index e4cbc495875..be0165d74bb 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h @@ -21,7 +21,7 @@ class TType; namespace NTable { class TQueryStats; class TTableDescription; -class TIndexDescription; +class TIndexDescription; } //! Provides access to raw protobuf values of YDB API entities. It is not recommended to use this @@ -37,20 +37,20 @@ public: static const Ydb::PersQueue::V1::DescribeTopicResult& GetProto(const NYdb::NPersQueue::TDescribeTopicResult& topicDescription); static NTable::TQueryStats FromProto(const Ydb::TableStats::QueryStats& queryStats); - static NTable::TTableDescription FromProto(const Ydb::Table::CreateTableRequest& request); - static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndex& tableIndex); - static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc); - - static Ydb::Table::ValueSinceUnixEpochModeSettings::Unit GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value); - static NTable::TValueSinceUnixEpochModeSettings::EUnit FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value); - - // exports & imports - template <typename TProtoSettings> static typename TProtoSettings::Scheme GetProto(ES3Scheme value); - template <typename TProtoSettings> static ES3Scheme FromProto(typename TProtoSettings::Scheme value); - static Ydb::Export::ExportToS3Settings::StorageClass GetProto(NExport::TExportToS3Settings::EStorageClass value); - static NExport::TExportToS3Settings::EStorageClass FromProto(Ydb::Export::ExportToS3Settings::StorageClass value); - static NExport::EExportProgress FromProto(Ydb::Export::ExportProgress::Progress value); - static NImport::EImportProgress FromProto(Ydb::Import::ImportProgress::Progress value); + static NTable::TTableDescription FromProto(const Ydb::Table::CreateTableRequest& request); + static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndex& tableIndex); + static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc); + + static Ydb::Table::ValueSinceUnixEpochModeSettings::Unit GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value); + static NTable::TValueSinceUnixEpochModeSettings::EUnit FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value); + + // exports & imports + template <typename TProtoSettings> static typename TProtoSettings::Scheme GetProto(ES3Scheme value); + template <typename TProtoSettings> static ES3Scheme FromProto(typename TProtoSettings::Scheme value); + static Ydb::Export::ExportToS3Settings::StorageClass GetProto(NExport::TExportToS3Settings::EStorageClass value); + static NExport::TExportToS3Settings::EStorageClass FromProto(Ydb::Export::ExportToS3Settings::StorageClass value); + static NExport::EExportProgress FromProto(Ydb::Export::ExportProgress::Progress value); + static NImport::EImportProgress FromProto(Ydb::Import::ImportProgress::Progress value); }; } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.cpp b/ydb/public/sdk/cpp/client/ydb_result/result.cpp index 24043ae326a..dc7eb09c0ac 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/result.cpp @@ -12,19 +12,19 @@ namespace NYdb { -TString TColumn::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TColumn::Out(IOutputStream& o) const { - o << "{ name: \"" << Name << "\"" - << ", type: " << Type - << " }"; -} - +TString TColumn::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TColumn::Out(IOutputStream& o) const { + o << "{ name: \"" << Name << "\"" + << ", type: " << Type + << " }"; +} + bool operator==(const TColumn& col1, const TColumn& col2) { return col1.Name == col2.Name && TypesEqual(col1.Type, col2.Type); } diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.h b/ydb/public/sdk/cpp/client/ydb_result/result.h index e5b84a455d3..287c8078be0 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.h +++ b/ydb/public/sdk/cpp/client/ydb_result/result.h @@ -20,9 +20,9 @@ struct TColumn { TColumn(const TString& name, const TType& type) : Name(name) , Type(type) {} - - TString ToString() const; - void Out(IOutputStream& o) const; + + TString ToString() const; + void Out(IOutputStream& o) const; }; bool operator==(const TColumn& col1, const TColumn& col2); @@ -107,7 +107,7 @@ private: using TResultSets = TVector<TResultSet>; } // namespace NYdb - -Y_DECLARE_OUT_SPEC(inline, NYdb::TColumn, o, x) { - return x.Out(o); -} + +Y_DECLARE_OUT_SPEC(inline, NYdb::TColumn, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp index 36c14ee8295..31f5535b968 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp @@ -33,8 +33,8 @@ static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) return ESchemeEntryType::CoordinationNode; case ::Ydb::Scheme::Entry::SEQUENCE: return ESchemeEntryType::Sequence; - case ::Ydb::Scheme::Entry::REPLICATION: - return ESchemeEntryType::Replication; + case ::Ydb::Scheme::Entry::REPLICATION: + return ESchemeEntryType::Replication; default: return ESchemeEntryType::Unknown; } @@ -82,7 +82,7 @@ public: entry.Name = result.self().name(); entry.Owner = result.self().owner(); entry.Type = ConvertProtoEntryType(result.self().type()); - entry.SizeBytes = result.self().size_bytes(); + entry.SizeBytes = result.self().size_bytes(); PermissionToSchemeEntry(result.self().effective_permissions(), &entry.EffectivePermissions); PermissionToSchemeEntry(result.self().permissions(), &entry.Permissions); } diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h index f32d51a9ccc..9d762b14c6a 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.h @@ -29,7 +29,7 @@ enum class ESchemeEntryType : i32 { BlockStoreVolume = 6, CoordinationNode = 7, Sequence = 15, - Replication = 16, + Replication = 16, }; struct TSchemeEntry { @@ -38,7 +38,7 @@ struct TSchemeEntry { ESchemeEntryType Type; TVector<TPermissions> EffectivePermissions; TVector<TPermissions> Permissions; - ui64 SizeBytes = 0; + ui64 SizeBytes = 0; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp index c64dc8060a8..40bfe74f67b 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp @@ -20,42 +20,42 @@ NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTabl return NTable::TTableDescription(request); } -NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex& tableIndex) { - return NTable::TIndexDescription(tableIndex); -} - -NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc) { - return NTable::TIndexDescription(tableIndexDesc); -} - -Ydb::Table::ValueSinceUnixEpochModeSettings::Unit TProtoAccessor::GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value) { - switch (value) { - case NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds: - return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS; - case NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds: - return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS; - case NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds: - return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS; - case NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds: - return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS; - case NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown: - return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_UNSPECIFIED; - } -} - -NTable::TValueSinceUnixEpochModeSettings::EUnit TProtoAccessor::FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value) { - switch (value) { - case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS: - return NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds; - case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS: - return NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds; - case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS: - return NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds; - case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS: - return NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds; - default: - return NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown; - } -} - -} +NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex& tableIndex) { + return NTable::TIndexDescription(tableIndex); +} + +NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc) { + return NTable::TIndexDescription(tableIndexDesc); +} + +Ydb::Table::ValueSinceUnixEpochModeSettings::Unit TProtoAccessor::GetProto(NTable::TValueSinceUnixEpochModeSettings::EUnit value) { + switch (value) { + case NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS; + case NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown: + return Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_UNSPECIFIED; + } +} + +NTable::TValueSinceUnixEpochModeSettings::EUnit TProtoAccessor::FromProto(Ydb::Table::ValueSinceUnixEpochModeSettings::Unit value) { + switch (value) { + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_SECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::Seconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::MilliSeconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MICROSECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::MicroSeconds; + case Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_NANOSECONDS: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::NanoSeconds; + default: + return NTable::TValueSinceUnixEpochModeSettings::EUnit::Unknown; + } +} + +} diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index c90154bbfe8..610fec38847 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -19,7 +19,7 @@ #include <util/generic/map.h> #include <util/random/random.h> -#include <util/string/join.h> +#include <util/string/join.h> #include <unordered_map> @@ -164,7 +164,7 @@ TBuildIndexOperation::TBuildIndexOperation(TStatus &&status, Ydb::Operations::Op Metadata_.Progress = metadata.progress(); const auto& desc = metadata.description(); Metadata_.Path = desc.path(); - Metadata_.Desctiption = TProtoAccessor::FromProto(desc.index()); + Metadata_.Desctiption = TProtoAccessor::FromProto(desc.index()); } const TBuildIndexOperation::TMetadata& TBuildIndexOperation::Metadata() const { @@ -248,100 +248,100 @@ static TInstant ProtobufTimestampToTInstant(const NProtoBuf::Timestamp& timestam } class TTableDescription::TImpl { - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - - template <typename TProto> - TImpl(const TProto& proto) - : StorageSettings_(proto.storage_settings()) - , PartitioningSettings_(proto.partitioning_settings()) - , HasStorageSettings_(proto.has_storage_settings()) - , HasPartitioningSettings_(proto.has_partitioning_settings()) + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + template <typename TProto> + TImpl(const TProto& proto) + : StorageSettings_(proto.storage_settings()) + , PartitioningSettings_(proto.partitioning_settings()) + , HasStorageSettings_(proto.has_storage_settings()) + , HasPartitioningSettings_(proto.has_partitioning_settings()) { - // primary key - for (const auto& pk : proto.primary_key()) { + // primary key + for (const auto& pk : proto.primary_key()) { PrimaryKey_.push_back(pk); } - - // columns - for (const auto& col : proto.columns()) { + + // columns + for (const auto& col : proto.columns()) { Columns_.emplace_back(col.name(), col.type(), col.family()); } - - // indexes - Indexes_.reserve(proto.indexesSize()); - for (const auto& index : proto.indexes()) { - Indexes_.emplace_back(TProtoAccessor::FromProto(index)); + + // indexes + Indexes_.reserve(proto.indexesSize()); + for (const auto& index : proto.indexes()) { + Indexes_.emplace_back(TProtoAccessor::FromProto(index)); + } + + // ttl settings + switch (proto.ttl_settings().mode_case()) { + case Ydb::Table::TtlSettings::kDateTypeColumn: + TtlSettings_ = TTtlSettings( + proto.ttl_settings().date_type_column(), + proto.ttl_settings().run_interval_seconds() + ); + break; + + case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: + TtlSettings_ = TTtlSettings( + proto.ttl_settings().value_since_unix_epoch(), + proto.ttl_settings().run_interval_seconds() + ); + break; + + default: + break; + } + + // column families + ColumnFamilies_.reserve(proto.column_families_size()); + for (const auto& family : proto.column_families()) { + ColumnFamilies_.emplace_back(family); + } + + // attributes + for (auto [key, value] : proto.attributes()) { + Attributes_[key] = value; + } + + // key bloom filter + switch (proto.key_bloom_filter()) { + case Ydb::FeatureFlag::ENABLED: + KeyBloomFilter_ = true; + break; + case Ydb::FeatureFlag::DISABLED: + KeyBloomFilter_ = false; + break; + default: + break; + } + + // read replicas settings + if (proto.has_read_replicas_settings()) { + const auto settings = proto.read_replicas_settings(); + switch (settings.settings_case()) { + case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: + ReadReplicasSettings_ = TReadReplicasSettings( + TReadReplicasSettings::EMode::PerAz, + settings.per_az_read_replicas_count()); + break; + case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: + ReadReplicasSettings_ = TReadReplicasSettings( + TReadReplicasSettings::EMode::AnyAz, + settings.any_az_read_replicas_count()); + break; + default: + break; + } } - - // ttl settings - switch (proto.ttl_settings().mode_case()) { - case Ydb::Table::TtlSettings::kDateTypeColumn: - TtlSettings_ = TTtlSettings( - proto.ttl_settings().date_type_column(), - proto.ttl_settings().run_interval_seconds() - ); - break; - - case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: - TtlSettings_ = TTtlSettings( - proto.ttl_settings().value_since_unix_epoch(), - proto.ttl_settings().run_interval_seconds() - ); - break; - - default: - break; - } - - // column families - ColumnFamilies_.reserve(proto.column_families_size()); - for (const auto& family : proto.column_families()) { - ColumnFamilies_.emplace_back(family); - } - - // attributes - for (auto [key, value] : proto.attributes()) { - Attributes_[key] = value; - } - - // key bloom filter - switch (proto.key_bloom_filter()) { - case Ydb::FeatureFlag::ENABLED: - KeyBloomFilter_ = true; - break; - case Ydb::FeatureFlag::DISABLED: - KeyBloomFilter_ = false; - break; - default: - break; - } - - // read replicas settings - if (proto.has_read_replicas_settings()) { - const auto settings = proto.read_replicas_settings(); - switch (settings.settings_case()) { - case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: - ReadReplicasSettings_ = TReadReplicasSettings( - TReadReplicasSettings::EMode::PerAz, - settings.per_az_read_replicas_count()); - break; - case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: - ReadReplicasSettings_ = TReadReplicasSettings( - TReadReplicasSettings::EMode::AnyAz, - settings.any_az_read_replicas_count()); - break; - default: - break; - } - } - } - -public: - TImpl(Ydb::Table::DescribeTableResult&& desc, const TDescribeTableSettings& describeSettings) - : TImpl(desc) - { - Proto_ = std::move(desc); - + } + +public: + TImpl(Ydb::Table::DescribeTableResult&& desc, const TDescribeTableSettings& describeSettings) + : TImpl(desc) + { + Proto_ = std::move(desc); + Owner_ = Proto_.self().owner(); PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); @@ -377,33 +377,33 @@ public: leftValue ? TKeyBound::Inclusive(*leftValue) : TMaybe<TKeyBound>(), TMaybe<TKeyBound>())); } - } - - struct TCreateTableRequestTag {}; // to avoid delegation cycle - - TImpl(const Ydb::Table::CreateTableRequest& request, TCreateTableRequestTag) - : TImpl(request) - { - if (request.compaction_policy()) { - SetCompactionPolicy(request.compaction_policy()); - } - - switch (request.partitions_case()) { - case Ydb::Table::CreateTableRequest::kUniformPartitions: - SetUniformPartitions(request.uniform_partitions()); - break; - - case Ydb::Table::CreateTableRequest::kPartitionAtKeys: { - TExplicitPartitions partitionAtKeys; - for (const auto& splitPoint : request.partition_at_keys().split_points()) { - TValue value(TType(splitPoint.type()), splitPoint.value()); - partitionAtKeys.AppendSplitPoints(value); - } - - SetPartitionAtKeys(partitionAtKeys); + } + + struct TCreateTableRequestTag {}; // to avoid delegation cycle + + TImpl(const Ydb::Table::CreateTableRequest& request, TCreateTableRequestTag) + : TImpl(request) + { + if (request.compaction_policy()) { + SetCompactionPolicy(request.compaction_policy()); + } + + switch (request.partitions_case()) { + case Ydb::Table::CreateTableRequest::kUniformPartitions: + SetUniformPartitions(request.uniform_partitions()); break; - } - + + case Ydb::Table::CreateTableRequest::kPartitionAtKeys: { + TExplicitPartitions partitionAtKeys; + for (const auto& splitPoint : request.partition_at_keys().split_points()) { + TValue value(TType(splitPoint.type()), splitPoint.value()); + partitionAtKeys.AppendSplitPoints(value); + } + + SetPartitionAtKeys(partitionAtKeys); + break; + } + default: break; } @@ -423,22 +423,22 @@ public: PrimaryKey_ = primaryKeyColumns; } - void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { - Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns)); + void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns)); } - void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); + void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); + } + + void SetTtlSettings(TTtlSettings&& settings) { + TtlSettings_ = std::move(settings); + } + + void SetTtlSettings(const TTtlSettings& settings) { + TtlSettings_ = settings; } - void SetTtlSettings(TTtlSettings&& settings) { - TtlSettings_ = std::move(settings); - } - - void SetTtlSettings(const TTtlSettings& settings) { - TtlSettings_ = settings; - } - void SetStorageSettings(const TStorageSettings& settings) { StorageSettings_ = settings; HasStorageSettings_ = true; @@ -448,18 +448,18 @@ public: ColumnFamilies_.emplace_back(desc); } - void AddAttribute(const TString& key, const TString& value) { - Attributes_[key] = value; - } - - void SetAttributes(const THashMap<TString, TString>& attrs) { - Attributes_ = attrs; - } - - void SetAttributes(THashMap<TString, TString>&& attrs) { - Attributes_ = std::move(attrs); - } - + void AddAttribute(const TString& key, const TString& value) { + Attributes_[key] = value; + } + + void SetAttributes(const THashMap<TString, TString>& attrs) { + Attributes_ = attrs; + } + + void SetAttributes(THashMap<TString, TString>&& attrs) { + Attributes_ = std::move(attrs); + } + void SetCompactionPolicy(const TString& name) { CompactionPolicy_ = name; } @@ -497,10 +497,10 @@ public: return Indexes_; } - const TMaybe<TTtlSettings>& GetTtlSettings() const { - return TtlSettings_; - } - + const TMaybe<TTtlSettings>& GetTtlSettings() const { + return TtlSettings_; + } + const TString& GetOwner() const { return Owner_; } @@ -525,10 +525,10 @@ public: return TableStats; } - bool HasStorageSettings() const { - return HasStorageSettings_; - } - + bool HasStorageSettings() const { + return HasStorageSettings_; + } + const TStorageSettings& GetStorageSettings() const { return StorageSettings_; } @@ -537,26 +537,26 @@ public: return ColumnFamilies_; } - const THashMap<TString, TString>& GetAttributes() const { - return Attributes_; - } - - const TString& GetCompactionPolicy() const { - return CompactionPolicy_; - } - - const TMaybe<ui64>& GetUniformPartitions() const { - return UniformPartitions_; - } - - const TMaybe<TExplicitPartitions>& GetPartitionAtKeys() const { - return PartitionAtKeys_; - } - - bool HasPartitioningSettings() const { - return HasPartitioningSettings_; - } - + const THashMap<TString, TString>& GetAttributes() const { + return Attributes_; + } + + const TString& GetCompactionPolicy() const { + return CompactionPolicy_; + } + + const TMaybe<ui64>& GetUniformPartitions() const { + return UniformPartitions_; + } + + const TMaybe<TExplicitPartitions>& GetPartitionAtKeys() const { + return PartitionAtKeys_; + } + + bool HasPartitioningSettings() const { + return HasPartitioningSettings_; + } + const TPartitioningSettings& GetPartitioningSettings() const { return PartitioningSettings_; } @@ -575,7 +575,7 @@ private: TVector<TString> PrimaryKey_; TVector<TTableColumn> Columns_; TVector<TIndexDescription> Indexes_; - TMaybe<TTtlSettings> TtlSettings_; + TMaybe<TTtlSettings> TtlSettings_; TString Owner_; TVector<NScheme::TPermissions> Permissions_; TVector<NScheme::TPermissions> EffectivePermissions_; @@ -583,7 +583,7 @@ private: TVector<TPartitionStats> PartitionStats_; TTableStats TableStats; TVector<TColumnFamilyDescription> ColumnFamilies_; - THashMap<TString, TString> Attributes_; + THashMap<TString, TString> Attributes_; TString CompactionPolicy_; TMaybe<ui64> UniformPartitions_; TMaybe<TExplicitPartitions> PartitionAtKeys_; @@ -605,11 +605,11 @@ TTableDescription::TTableDescription(Ydb::Table::DescribeTableResult&& desc, { } -TTableDescription::TTableDescription(const Ydb::Table::CreateTableRequest& request) - : Impl_(new TImpl(request, TImpl::TCreateTableRequestTag())) -{ -} - +TTableDescription::TTableDescription(const Ydb::Table::CreateTableRequest& request) + : Impl_(new TImpl(request, TImpl::TCreateTableRequestTag())) +{ +} + const TVector<TString>& TTableDescription::GetPrimaryKeyColumns() const { return Impl_->GetPrimaryKeyColumns(); } @@ -632,10 +632,10 @@ TVector<TIndexDescription> TTableDescription::GetIndexDescriptions() const { return Impl_->GetIndexDescriptions(); } -TMaybe<TTtlSettings> TTableDescription::GetTtlSettings() const { - return Impl_->GetTtlSettings(); -} - +TMaybe<TTtlSettings> TTableDescription::GetTtlSettings() const { + return Impl_->GetTtlSettings(); +} + const TString& TTableDescription::GetOwner() const { return Impl_->GetOwner(); } @@ -660,46 +660,46 @@ void TTableDescription::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyC Impl_->SetPrimaryKeyColumns(primaryKeyColumns); } -void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { - Impl_->AddSecondaryIndex(indexName, type, indexColumns); -} - -void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); -} - -void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); -} - -void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); -} - -void TTableDescription::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); -} - -void TTableDescription::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); -} - -void TTableDescription::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - AddSyncSecondaryIndex(indexName, indexColumns); -} - -void TTableDescription::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); -} - -void TTableDescription::SetTtlSettings(TTtlSettings&& settings) { - Impl_->SetTtlSettings(std::move(settings)); -} - -void TTableDescription::SetTtlSettings(const TTtlSettings& settings) { - Impl_->SetTtlSettings(settings); -} - +void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { + Impl_->AddSecondaryIndex(indexName, type, indexColumns); +} + +void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); +} + +void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); +} + +void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); +} + +void TTableDescription::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); +} + +void TTableDescription::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); +} + +void TTableDescription::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + AddSyncSecondaryIndex(indexName, indexColumns); +} + +void TTableDescription::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); +} + +void TTableDescription::SetTtlSettings(TTtlSettings&& settings) { + Impl_->SetTtlSettings(std::move(settings)); +} + +void TTableDescription::SetTtlSettings(const TTtlSettings& settings) { + Impl_->SetTtlSettings(settings); +} + void TTableDescription::SetStorageSettings(const TStorageSettings& settings) { Impl_->SetStorageSettings(settings); } @@ -708,18 +708,18 @@ void TTableDescription::AddColumnFamily(const TColumnFamilyDescription& desc) { Impl_->AddColumnFamily(desc); } -void TTableDescription::AddAttribute(const TString& key, const TString& value) { - Impl_->AddAttribute(key, value); -} - -void TTableDescription::SetAttributes(const THashMap<TString, TString>& attrs) { - Impl_->SetAttributes(attrs); -} - -void TTableDescription::SetAttributes(THashMap<TString, TString>&& attrs) { - Impl_->SetAttributes(std::move(attrs)); -} - +void TTableDescription::AddAttribute(const TString& key, const TString& value) { + Impl_->AddAttribute(key, value); +} + +void TTableDescription::SetAttributes(const THashMap<TString, TString>& attrs) { + Impl_->SetAttributes(attrs); +} + +void TTableDescription::SetAttributes(THashMap<TString, TString>&& attrs) { + Impl_->SetAttributes(std::move(attrs)); +} + void TTableDescription::SetCompactionPolicy(const TString& name) { Impl_->SetCompactionPolicy(name); } @@ -776,10 +776,10 @@ const TVector<TColumnFamilyDescription>& TTableDescription::GetColumnFamilies() return Impl_->GetColumnFamilies(); } -const THashMap<TString, TString>& TTableDescription::GetAttributes() const { - return Impl_->GetAttributes(); -} - +const THashMap<TString, TString>& TTableDescription::GetAttributes() const { + return Impl_->GetAttributes(); +} + const TPartitioningSettings& TTableDescription::GetPartitioningSettings() const { return Impl_->GetPartitioningSettings(); } @@ -796,84 +796,84 @@ const Ydb::Table::DescribeTableResult& TTableDescription::GetProto() const { return Impl_->GetProto(); } -void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) const { - for (const auto& column : Impl_->GetColumns()) { - auto& protoColumn = *request.add_columns(); - protoColumn.set_name(column.Name); - protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); - protoColumn.set_family(column.Family); - } - - for (const auto& pk : Impl_->GetPrimaryKeyColumns()) { - request.add_primary_key(pk); - } - - for (const auto& index : Impl_->GetIndexDescriptions()) { - index.SerializeTo(*request.add_indexes()); - } - - if (const auto& ttl = Impl_->GetTtlSettings()) { - ttl->SerializeTo(*request.mutable_ttl_settings()); - } - - if (Impl_->HasStorageSettings()) { - request.mutable_storage_settings()->CopyFrom(Impl_->GetStorageSettings().GetProto()); - } - - for (const auto& family : Impl_->GetColumnFamilies()) { - auto* f = request.add_column_families(); - f->CopyFrom(family.GetProto()); - } - - for (const auto& [key, value] : Impl_->GetAttributes()) { - (*request.mutable_attributes())[key] = value; - } - - if (Impl_->GetCompactionPolicy()) { - request.set_compaction_policy(Impl_->GetCompactionPolicy()); - } - - if (const auto& uniformPartitions = Impl_->GetUniformPartitions()) { - request.set_uniform_partitions(uniformPartitions.GetRef()); - } - - if (const auto& partitionAtKeys = Impl_->GetPartitionAtKeys()) { - auto* borders = request.mutable_partition_at_keys(); - for (const auto& splitPoint : partitionAtKeys->SplitPoints_) { - auto* border = borders->Addsplit_points(); - border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); - border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); - } - } else if (Impl_->GetProto().shard_key_bounds_size()) { - request.mutable_partition_at_keys()->mutable_split_points()->CopyFrom(Impl_->GetProto().shard_key_bounds()); - } - - if (Impl_->HasPartitioningSettings()) { - request.mutable_partitioning_settings()->CopyFrom(Impl_->GetPartitioningSettings().GetProto()); - } - - if (auto keyBloomFilter = Impl_->GetKeyBloomFilter()) { - if (keyBloomFilter.GetRef()) { - request.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); - } else { - request.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); - } - } - - if (const auto& settings = Impl_->GetReadReplicasSettings()) { - switch (settings->GetMode()) { - case TReadReplicasSettings::EMode::PerAz: - request.mutable_read_replicas_settings()->set_per_az_read_replicas_count(settings->GetReadReplicasCount()); - break; - case TReadReplicasSettings::EMode::AnyAz: - request.mutable_read_replicas_settings()->set_any_az_read_replicas_count(settings->GetReadReplicasCount()); - break; - default: - break; - } - } -} - +void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) const { + for (const auto& column : Impl_->GetColumns()) { + auto& protoColumn = *request.add_columns(); + protoColumn.set_name(column.Name); + protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); + protoColumn.set_family(column.Family); + } + + for (const auto& pk : Impl_->GetPrimaryKeyColumns()) { + request.add_primary_key(pk); + } + + for (const auto& index : Impl_->GetIndexDescriptions()) { + index.SerializeTo(*request.add_indexes()); + } + + if (const auto& ttl = Impl_->GetTtlSettings()) { + ttl->SerializeTo(*request.mutable_ttl_settings()); + } + + if (Impl_->HasStorageSettings()) { + request.mutable_storage_settings()->CopyFrom(Impl_->GetStorageSettings().GetProto()); + } + + for (const auto& family : Impl_->GetColumnFamilies()) { + auto* f = request.add_column_families(); + f->CopyFrom(family.GetProto()); + } + + for (const auto& [key, value] : Impl_->GetAttributes()) { + (*request.mutable_attributes())[key] = value; + } + + if (Impl_->GetCompactionPolicy()) { + request.set_compaction_policy(Impl_->GetCompactionPolicy()); + } + + if (const auto& uniformPartitions = Impl_->GetUniformPartitions()) { + request.set_uniform_partitions(uniformPartitions.GetRef()); + } + + if (const auto& partitionAtKeys = Impl_->GetPartitionAtKeys()) { + auto* borders = request.mutable_partition_at_keys(); + for (const auto& splitPoint : partitionAtKeys->SplitPoints_) { + auto* border = borders->Addsplit_points(); + border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); + border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); + } + } else if (Impl_->GetProto().shard_key_bounds_size()) { + request.mutable_partition_at_keys()->mutable_split_points()->CopyFrom(Impl_->GetProto().shard_key_bounds()); + } + + if (Impl_->HasPartitioningSettings()) { + request.mutable_partitioning_settings()->CopyFrom(Impl_->GetPartitioningSettings().GetProto()); + } + + if (auto keyBloomFilter = Impl_->GetKeyBloomFilter()) { + if (keyBloomFilter.GetRef()) { + request.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); + } else { + request.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); + } + } + + if (const auto& settings = Impl_->GetReadReplicasSettings()) { + switch (settings->GetMode()) { + case TReadReplicasSettings::EMode::PerAz: + request.mutable_read_replicas_settings()->set_per_az_read_replicas_count(settings->GetReadReplicasCount()); + break; + case TReadReplicasSettings::EMode::AnyAz: + request.mutable_read_replicas_settings()->set_any_az_read_replicas_count(settings->GetReadReplicasCount()); + break; + default: + break; + } + } +} + //////////////////////////////////////////////////////////////////////////////// class TStorageSettingsBuilder::TImpl { @@ -1043,75 +1043,75 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum return *this; } -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); return *this; } -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { - TableDescription_.AddSecondaryIndex(indexName, type, indexColumns); +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns) { + TableDescription_.AddSecondaryIndex(indexName, type, indexColumns); + return *this; +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn) { + TableDescription_.AddSecondaryIndex(indexName, type, TVector<TString>{indexColumn}); + return *this; +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); +} + +TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TString& indexColumn) { + return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumn); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); +} + +TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TString& indexColumn) { + return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumn); +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { + return AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { + return AddSyncSecondaryIndex(indexName, indexColumns); +} + +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TString& indexColumn) { + return AddSyncSecondaryIndex(indexName, indexColumn); +} + +TTableBuilder& TTableBuilder::SetTtlSettings(TTtlSettings&& settings) { + TableDescription_.SetTtlSettings(std::move(settings)); return *this; } -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn) { - TableDescription_.AddSecondaryIndex(indexName, type, TVector<TString>{indexColumn}); +TTableBuilder& TTableBuilder::SetTtlSettings(const TTtlSettings& settings) { + TableDescription_.SetTtlSettings(settings); return *this; } -TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns, dataColumns); -} - -TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); -} - -TTableBuilder& TTableBuilder::AddSyncSecondaryIndex(const TString& indexName, const TString& indexColumn) { - return AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumn); -} - -TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns, dataColumns); -} - -TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumns); -} - -TTableBuilder& TTableBuilder::AddAsyncSecondaryIndex(const TString& indexName, const TString& indexColumn) { - return AddSecondaryIndex(indexName, EIndexType::GlobalAsync, indexColumn); -} - -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { - return AddSyncSecondaryIndex(indexName, indexColumns, dataColumns); -} - -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) { - return AddSyncSecondaryIndex(indexName, indexColumns); -} - -TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, const TString& indexColumn) { - return AddSyncSecondaryIndex(indexName, indexColumn); -} - -TTableBuilder& TTableBuilder::SetTtlSettings(TTtlSettings&& settings) { - TableDescription_.SetTtlSettings(std::move(settings)); - return *this; -} - -TTableBuilder& TTableBuilder::SetTtlSettings(const TTtlSettings& settings) { - TableDescription_.SetTtlSettings(settings); - return *this; -} - -TTableBuilder& TTableBuilder::SetTtlSettings(const TString& columnName, const TDuration& expireAfter) { - return SetTtlSettings(TTtlSettings(columnName, expireAfter)); -} - -TTableBuilder& TTableBuilder::SetTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) { - return SetTtlSettings(TTtlSettings(columnName, columnUnit, expireAfter)); -} - +TTableBuilder& TTableBuilder::SetTtlSettings(const TString& columnName, const TDuration& expireAfter) { + return SetTtlSettings(TTtlSettings(columnName, expireAfter)); +} + +TTableBuilder& TTableBuilder::SetTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) { + return SetTtlSettings(TTtlSettings(columnName, columnUnit, expireAfter)); +} + TTableBuilder& TTableBuilder::SetStorageSettings(const TStorageSettings& settings) { TableDescription_.SetStorageSettings(settings); return *this; @@ -1122,21 +1122,21 @@ TTableBuilder& TTableBuilder::AddColumnFamily(const TColumnFamilyDescription& de return *this; } -TTableBuilder& TTableBuilder::AddAttribute(const TString& key, const TString& value) { - TableDescription_.AddAttribute(key, value); - return *this; -} - -TTableBuilder& TTableBuilder::SetAttributes(const THashMap<TString, TString>& attrs) { - TableDescription_.SetAttributes(attrs); - return *this; -} - -TTableBuilder& TTableBuilder::SetAttributes(THashMap<TString, TString>&& attrs) { - TableDescription_.SetAttributes(std::move(attrs)); - return *this; -} - +TTableBuilder& TTableBuilder::AddAttribute(const TString& key, const TString& value) { + TableDescription_.AddAttribute(key, value); + return *this; +} + +TTableBuilder& TTableBuilder::SetAttributes(const THashMap<TString, TString>& attrs) { + TableDescription_.SetAttributes(attrs); + return *this; +} + +TTableBuilder& TTableBuilder::SetAttributes(THashMap<TString, TString>&& attrs) { + TableDescription_.SetAttributes(std::move(attrs)); + return *this; +} + TTableBuilder& TTableBuilder::SetCompactionPolicy(const TString& name) { TableDescription_.SetCompactionPolicy(name); return *this; @@ -3684,7 +3684,7 @@ TFuture<TStatus> TSession::CreateTable(const TString& path, TTableDescription&& request.set_session_id(SessionImpl_->GetId()); request.set_path(path); - tableDesc.SerializeTo(request); + tableDesc.SerializeTo(request); ConvertCreateTableSettingsToProto(settings, request.mutable_profile()); @@ -3728,7 +3728,7 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( } for (const auto& addIndex : settings.AddIndexes_) { - addIndex.SerializeTo(*request.add_add_indexes()); + addIndex.SerializeTo(*request.add_add_indexes()); } for (const auto& dropIndex : settings.DropIndexes_) { @@ -3747,21 +3747,21 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( request.add_alter_column_families()->CopyFrom(family.GetProto()); } - if (const auto& ttl = settings.GetAlterTtlSettings()) { - switch (ttl->GetAction()) { - case TAlterTtlSettings::EAction::Set: - ttl->GetTtlSettings().SerializeTo(*request.mutable_set_ttl_settings()); - break; - case TAlterTtlSettings::EAction::Drop: - request.mutable_drop_ttl_settings(); - break; - } - } - - for (const auto& [key, value] : settings.AlterAttributes_) { - (*request.mutable_alter_attributes())[key] = value; - } - + if (const auto& ttl = settings.GetAlterTtlSettings()) { + switch (ttl->GetAction()) { + case TAlterTtlSettings::EAction::Set: + ttl->GetTtlSettings().SerializeTo(*request.mutable_set_ttl_settings()); + break; + case TAlterTtlSettings::EAction::Drop: + request.mutable_drop_ttl_settings(); + break; + } + } + + for (const auto& [key, value] : settings.AlterAttributes_) { + (*request.mutable_alter_attributes())[key] = value; + } + if (settings.SetCompactionPolicy_) { request.set_set_compaction_policy(settings.SetCompactionPolicy_); } @@ -4297,34 +4297,34 @@ bool TRenameItem::ReplaceDestination() const { //////////////////////////////////////////////////////////////////////////////// -TIndexDescription::TIndexDescription(const TString& name, EIndexType type, - const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) +TIndexDescription::TIndexDescription(const TString& name, EIndexType type, + const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) : IndexName_(name) - , IndexType_(type) + , IndexType_(type) , IndexColumns_(indexColumns) - , DataColumns_(dataColumns) + , DataColumns_(dataColumns) +{} + +TIndexDescription::TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) + : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns) +{} + +TIndexDescription::TIndexDescription(const Ydb::Table::TableIndex& tableIndex) + : TIndexDescription(FromProto(tableIndex)) +{} + +TIndexDescription::TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc) + : TIndexDescription(FromProto(tableIndexDesc)) {} -TIndexDescription::TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) - : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns) -{} - -TIndexDescription::TIndexDescription(const Ydb::Table::TableIndex& tableIndex) - : TIndexDescription(FromProto(tableIndex)) -{} - -TIndexDescription::TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc) - : TIndexDescription(FromProto(tableIndexDesc)) -{} - const TString& TIndexDescription::GetIndexName() const { return IndexName_; } -EIndexType TIndexDescription::GetIndexType() const { - return IndexType_; -} - +EIndexType TIndexDescription::GetIndexType() const { + return IndexType_; +} + const TVector<TString>& TIndexDescription::GetIndexColumns() const { return IndexColumns_; } @@ -4333,326 +4333,326 @@ const TVector<TString>& TIndexDescription::GetDataColumns() const { return DataColumns_; } -ui64 TIndexDescription::GetSizeBytes() const { - return SizeBytes; -} - -template <typename TProto> -TIndexDescription TIndexDescription::FromProto(const TProto& proto) { - EIndexType type; - TVector<TString> indexColumns; - TVector<TString> dataColumns; - - indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end()); +ui64 TIndexDescription::GetSizeBytes() const { + return SizeBytes; +} + +template <typename TProto> +TIndexDescription TIndexDescription::FromProto(const TProto& proto) { + EIndexType type; + TVector<TString> indexColumns; + TVector<TString> dataColumns; + + indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end()); dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end()); - switch (proto.type_case()) { - case TProto::kGlobalIndex: - type = EIndexType::GlobalSync; - break; - case TProto::kGlobalAsyncIndex: - type = EIndexType::GlobalAsync; - break; - default: // fallback to global sync - type = EIndexType::GlobalSync; - break; - } - - auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns); - if constexpr (std::is_same_v<TProto, Ydb::Table::TableIndexDescription>) { - result.SizeBytes = proto.size_bytes(); - } - - return result; -} - -void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { - proto.set_name(IndexName_); - for (const auto& indexCol : IndexColumns_) { - proto.add_index_columns(indexCol); - } - + switch (proto.type_case()) { + case TProto::kGlobalIndex: + type = EIndexType::GlobalSync; + break; + case TProto::kGlobalAsyncIndex: + type = EIndexType::GlobalAsync; + break; + default: // fallback to global sync + type = EIndexType::GlobalSync; + break; + } + + auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns); + if constexpr (std::is_same_v<TProto, Ydb::Table::TableIndexDescription>) { + result.SizeBytes = proto.size_bytes(); + } + + return result; +} + +void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { + proto.set_name(IndexName_); + for (const auto& indexCol : IndexColumns_) { + proto.add_index_columns(indexCol); + } + *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; - switch (IndexType_) { - case EIndexType::GlobalSync: + switch (IndexType_) { + case EIndexType::GlobalSync: *proto.mutable_global_index() = Ydb::Table::GlobalIndex(); - break; - case EIndexType::GlobalAsync: + break; + case EIndexType::GlobalAsync: *proto.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); - break; - case EIndexType::Unknown: - break; - } -} - -TString TIndexDescription::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TIndexDescription::Out(IOutputStream& o) const { - o << "{ name: \"" << IndexName_ << "\""; - o << ", type: " << IndexType_ << ""; - o << ", index_columns: [" << JoinSeq(", ", IndexColumns_) << "]"; - - if (DataColumns_) { - o << ", data_columns: [" << JoinSeq(", ", DataColumns_) << "]"; - } - - o << " }"; -} - -bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs) { - return lhs.GetIndexName() == rhs.GetIndexName() - && lhs.GetIndexType() == rhs.GetIndexType() - && lhs.GetIndexColumns() == rhs.GetIndexColumns() - && lhs.GetDataColumns() == rhs.GetDataColumns(); -} - -bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs) { - return !(lhs == rhs); -} - + break; + case EIndexType::Unknown: + break; + } +} + +TString TIndexDescription::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TIndexDescription::Out(IOutputStream& o) const { + o << "{ name: \"" << IndexName_ << "\""; + o << ", type: " << IndexType_ << ""; + o << ", index_columns: [" << JoinSeq(", ", IndexColumns_) << "]"; + + if (DataColumns_) { + o << ", data_columns: [" << JoinSeq(", ", DataColumns_) << "]"; + } + + o << " }"; +} + +bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs) { + return lhs.GetIndexName() == rhs.GetIndexName() + && lhs.GetIndexType() == rhs.GetIndexType() + && lhs.GetIndexColumns() == rhs.GetIndexColumns() + && lhs.GetDataColumns() == rhs.GetDataColumns(); +} + +bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs) { + return !(lhs == rhs); +} + //////////////////////////////////////////////////////////////////////////////// -TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter) - : ColumnName_(columnName) - , ExpireAfter_(expireAfter) -{} - -void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const { - proto.set_column_name(ColumnName_); - proto.set_expire_after_seconds(ExpireAfter_.Seconds()); -} - -const TString& TDateTypeColumnModeSettings::GetColumnName() const { - return ColumnName_; -} - -const TDuration& TDateTypeColumnModeSettings::GetExpireAfter() const { - return ExpireAfter_; -} - -TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) - : ColumnName_(columnName) - , ColumnUnit_(columnUnit) - , ExpireAfter_(expireAfter) -{} - -void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const { - proto.set_column_name(ColumnName_); - proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); - proto.set_expire_after_seconds(ExpireAfter_.Seconds()); -} - -const TString& TValueSinceUnixEpochModeSettings::GetColumnName() const { - return ColumnName_; -} - -TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::GetColumnUnit() const { - return ColumnUnit_; -} - -const TDuration& TValueSinceUnixEpochModeSettings::GetExpireAfter() const { - return ExpireAfter_; -} - -void TValueSinceUnixEpochModeSettings::Out(IOutputStream& out, EUnit unit) { -#define PRINT_UNIT(x) \ - case EUnit::x: \ - out << #x; \ - break - - switch (unit) { - PRINT_UNIT(Seconds); - PRINT_UNIT(MilliSeconds); - PRINT_UNIT(MicroSeconds); - PRINT_UNIT(NanoSeconds); - PRINT_UNIT(Unknown); - } - -#undef PRINT_UNIT -} - -TString TValueSinceUnixEpochModeSettings::ToString(EUnit unit) { - TString result; - TStringOutput out(result); - Out(out, unit); - return result; -} - -TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::UnitFromString(const TString& value) { - const auto norm = to_lower(value); - - if (norm == "s" || norm == "sec" || norm == "seconds") { - return EUnit::Seconds; - } else if (norm == "ms" || norm == "msec" || norm == "milliseconds") { - return EUnit::MilliSeconds; - } else if (norm == "us" || norm == "usec" || norm == "microseconds") { - return EUnit::MicroSeconds; - } else if (norm == "ns" || norm == "nsec" || norm == "nanoseconds") { - return EUnit::NanoSeconds; - } - - return EUnit::Unknown; -} - -TTtlSettings::TTtlSettings(const TString& columnName, const TDuration& expireAfter) - : Mode_(TDateTypeColumnModeSettings(columnName, expireAfter)) -{} - -TTtlSettings::TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds) - : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds())) -{ - RunInterval_ = TDuration::Seconds(runIntervalSeconds); -} - -const TDateTypeColumnModeSettings& TTtlSettings::GetDateTypeColumn() const { +TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter) + : ColumnName_(columnName) + , ExpireAfter_(expireAfter) +{} + +void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const { + proto.set_column_name(ColumnName_); + proto.set_expire_after_seconds(ExpireAfter_.Seconds()); +} + +const TString& TDateTypeColumnModeSettings::GetColumnName() const { + return ColumnName_; +} + +const TDuration& TDateTypeColumnModeSettings::GetExpireAfter() const { + return ExpireAfter_; +} + +TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) + : ColumnName_(columnName) + , ColumnUnit_(columnUnit) + , ExpireAfter_(expireAfter) +{} + +void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const { + proto.set_column_name(ColumnName_); + proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); + proto.set_expire_after_seconds(ExpireAfter_.Seconds()); +} + +const TString& TValueSinceUnixEpochModeSettings::GetColumnName() const { + return ColumnName_; +} + +TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::GetColumnUnit() const { + return ColumnUnit_; +} + +const TDuration& TValueSinceUnixEpochModeSettings::GetExpireAfter() const { + return ExpireAfter_; +} + +void TValueSinceUnixEpochModeSettings::Out(IOutputStream& out, EUnit unit) { +#define PRINT_UNIT(x) \ + case EUnit::x: \ + out << #x; \ + break + + switch (unit) { + PRINT_UNIT(Seconds); + PRINT_UNIT(MilliSeconds); + PRINT_UNIT(MicroSeconds); + PRINT_UNIT(NanoSeconds); + PRINT_UNIT(Unknown); + } + +#undef PRINT_UNIT +} + +TString TValueSinceUnixEpochModeSettings::ToString(EUnit unit) { + TString result; + TStringOutput out(result); + Out(out, unit); + return result; +} + +TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::UnitFromString(const TString& value) { + const auto norm = to_lower(value); + + if (norm == "s" || norm == "sec" || norm == "seconds") { + return EUnit::Seconds; + } else if (norm == "ms" || norm == "msec" || norm == "milliseconds") { + return EUnit::MilliSeconds; + } else if (norm == "us" || norm == "usec" || norm == "microseconds") { + return EUnit::MicroSeconds; + } else if (norm == "ns" || norm == "nsec" || norm == "nanoseconds") { + return EUnit::NanoSeconds; + } + + return EUnit::Unknown; +} + +TTtlSettings::TTtlSettings(const TString& columnName, const TDuration& expireAfter) + : Mode_(TDateTypeColumnModeSettings(columnName, expireAfter)) +{} + +TTtlSettings::TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds) + : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds())) +{ + RunInterval_ = TDuration::Seconds(runIntervalSeconds); +} + +const TDateTypeColumnModeSettings& TTtlSettings::GetDateTypeColumn() const { return std::get<TDateTypeColumnModeSettings>(Mode_); -} - -TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) - : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter)) -{} - -TTtlSettings::TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds) - : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds())) -{ - RunInterval_ = TDuration::Seconds(runIntervalSeconds); -} - -const TValueSinceUnixEpochModeSettings& TTtlSettings::GetValueSinceUnixEpoch() const { +} + +TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) + : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter)) +{} + +TTtlSettings::TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds) + : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds())) +{ + RunInterval_ = TDuration::Seconds(runIntervalSeconds); +} + +const TValueSinceUnixEpochModeSettings& TTtlSettings::GetValueSinceUnixEpoch() const { return std::get<TValueSinceUnixEpochModeSettings>(Mode_); -} - -void TTtlSettings::SerializeTo(Ydb::Table::TtlSettings& proto) const { - switch (GetMode()) { - case EMode::DateTypeColumn: - GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column()); - break; - case EMode::ValueSinceUnixEpoch: - GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch()); - break; - } - - if (RunInterval_) { - proto.set_run_interval_seconds(RunInterval_.Seconds()); - } -} - -TTtlSettings::EMode TTtlSettings::GetMode() const { - return static_cast<EMode>(Mode_.index()); -} - -TTtlSettings& TTtlSettings::SetRunInterval(const TDuration& value) { - RunInterval_ = value; - return *this; -} - -const TDuration& TTtlSettings::GetRunInterval() const { - return RunInterval_; -} - -TAlterTtlSettings::EAction TAlterTtlSettings::GetAction() const { - return static_cast<EAction>(Action_.index()); -} - -const TTtlSettings& TAlterTtlSettings::GetTtlSettings() const { +} + +void TTtlSettings::SerializeTo(Ydb::Table::TtlSettings& proto) const { + switch (GetMode()) { + case EMode::DateTypeColumn: + GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column()); + break; + case EMode::ValueSinceUnixEpoch: + GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch()); + break; + } + + if (RunInterval_) { + proto.set_run_interval_seconds(RunInterval_.Seconds()); + } +} + +TTtlSettings::EMode TTtlSettings::GetMode() const { + return static_cast<EMode>(Mode_.index()); +} + +TTtlSettings& TTtlSettings::SetRunInterval(const TDuration& value) { + RunInterval_ = value; + return *this; +} + +const TDuration& TTtlSettings::GetRunInterval() const { + return RunInterval_; +} + +TAlterTtlSettings::EAction TAlterTtlSettings::GetAction() const { + return static_cast<EAction>(Action_.index()); +} + +const TTtlSettings& TAlterTtlSettings::GetTtlSettings() const { return std::get<TTtlSettings>(Action_); -} - -class TAlterTtlSettingsBuilder::TImpl { - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - -public: - TImpl() { } - - void Drop() { - AlterTtlSettings_ = TAlterTtlSettings::Drop(); - } - - void Set(TTtlSettings&& settings) { - AlterTtlSettings_ = TAlterTtlSettings::Set(std::move(settings)); - } - - void Set(const TTtlSettings& settings) { - AlterTtlSettings_ = TAlterTtlSettings::Set(settings); - } - - const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const { - return AlterTtlSettings_; - } - -private: - TMaybe<TAlterTtlSettings> AlterTtlSettings_; -}; - -TAlterTtlSettingsBuilder::TAlterTtlSettingsBuilder(TAlterTableSettings& parent) - : Parent_(parent) - , Impl_(std::make_shared<TImpl>()) -{ } - -TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Drop() { - Impl_->Drop(); - return *this; -} - -TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(TTtlSettings&& settings) { - Impl_->Set(std::move(settings)); - return *this; -} - -TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TTtlSettings& settings) { - Impl_->Set(settings); - return *this; -} - -TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TString& columnName, const TDuration& expireAfter) { - return Set(TTtlSettings(columnName, expireAfter)); -} - -TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) { - return Set(TTtlSettings(columnName, columnUnit, expireAfter)); -} - -TAlterTableSettings& TAlterTtlSettingsBuilder::EndAlterTtlSettings() { - return Parent_.AlterTtlSettings(Impl_->GetAlterTtlSettings()); -} - -class TAlterTableSettings::TImpl { -public: - TImpl() { } - - void SetAlterTtlSettings(const TMaybe<TAlterTtlSettings>& value) { - AlterTtlSettings_ = value; - } - - const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const { - return AlterTtlSettings_; - } - -private: - TMaybe<TAlterTtlSettings> AlterTtlSettings_; -}; - -TAlterTableSettings::TAlterTableSettings() - : Impl_(std::make_shared<TImpl>()) -{ } - -TAlterTableSettings& TAlterTableSettings::AlterTtlSettings(const TMaybe<TAlterTtlSettings>& value) { - Impl_->SetAlterTtlSettings(value); - return *this; -} - -const TMaybe<TAlterTtlSettings>& TAlterTableSettings::GetAlterTtlSettings() const { - return Impl_->GetAlterTtlSettings(); -} - -//////////////////////////////////////////////////////////////////////////////// - +} + +class TAlterTtlSettingsBuilder::TImpl { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TImpl() { } + + void Drop() { + AlterTtlSettings_ = TAlterTtlSettings::Drop(); + } + + void Set(TTtlSettings&& settings) { + AlterTtlSettings_ = TAlterTtlSettings::Set(std::move(settings)); + } + + void Set(const TTtlSettings& settings) { + AlterTtlSettings_ = TAlterTtlSettings::Set(settings); + } + + const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const { + return AlterTtlSettings_; + } + +private: + TMaybe<TAlterTtlSettings> AlterTtlSettings_; +}; + +TAlterTtlSettingsBuilder::TAlterTtlSettingsBuilder(TAlterTableSettings& parent) + : Parent_(parent) + , Impl_(std::make_shared<TImpl>()) +{ } + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Drop() { + Impl_->Drop(); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(TTtlSettings&& settings) { + Impl_->Set(std::move(settings)); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TTtlSettings& settings) { + Impl_->Set(settings); + return *this; +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TString& columnName, const TDuration& expireAfter) { + return Set(TTtlSettings(columnName, expireAfter)); +} + +TAlterTtlSettingsBuilder& TAlterTtlSettingsBuilder::Set(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) { + return Set(TTtlSettings(columnName, columnUnit, expireAfter)); +} + +TAlterTableSettings& TAlterTtlSettingsBuilder::EndAlterTtlSettings() { + return Parent_.AlterTtlSettings(Impl_->GetAlterTtlSettings()); +} + +class TAlterTableSettings::TImpl { +public: + TImpl() { } + + void SetAlterTtlSettings(const TMaybe<TAlterTtlSettings>& value) { + AlterTtlSettings_ = value; + } + + const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const { + return AlterTtlSettings_; + } + +private: + TMaybe<TAlterTtlSettings> AlterTtlSettings_; +}; + +TAlterTableSettings::TAlterTableSettings() + : Impl_(std::make_shared<TImpl>()) +{ } + +TAlterTableSettings& TAlterTableSettings::AlterTtlSettings(const TMaybe<TAlterTtlSettings>& value) { + Impl_->SetAlterTtlSettings(value); + return *this; +} + +const TMaybe<TAlterTtlSettings>& TAlterTableSettings::GetAlterTtlSettings() const { + return Impl_->GetAlterTtlSettings(); +} + +//////////////////////////////////////////////////////////////////////////////// + TReadReplicasSettings::TReadReplicasSettings(EMode mode, ui64 readReplicasCount) : Mode_(mode) , ReadReplicasCount_(readReplicasCount) diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 002bdc0cbb8..42d21d2fec7 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -8,20 +8,20 @@ #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/maybe.h> -#include <util/generic/variant.h> +#include <util/generic/variant.h> namespace Ydb { namespace Table { class StorageSettings; class ColumnFamily; -class CreateTableRequest; +class CreateTableRequest; class DescribeTableResult; class PartitioningSettings; -class DateTypeColumnModeSettings; -class TtlSettings; +class DateTypeColumnModeSettings; +class TtlSettings; class TableIndex; class TableIndexDescription; class ValueSinceUnixEpochModeSettings; @@ -134,44 +134,44 @@ struct TAlterTableColumn { //! Represents index description class TIndexDescription { - friend class NYdb::TProtoAccessor; - + friend class NYdb::TProtoAccessor; + public: - TIndexDescription( - const TString& name, EIndexType type, - const TVector<TString>& indexColumns, - const TVector<TString>& dataColumns = TVector<TString>()); - + TIndexDescription( + const TString& name, EIndexType type, + const TVector<TString>& indexColumns, + const TVector<TString>& dataColumns = TVector<TString>()); + TIndexDescription(const TString& name, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns = TVector<TString>()); const TString& GetIndexName() const; - EIndexType GetIndexType() const; + EIndexType GetIndexType() const; const TVector<TString>& GetIndexColumns() const; const TVector<TString>& GetDataColumns() const; - ui64 GetSizeBytes() const; - - void SerializeTo(Ydb::Table::TableIndex& proto) const; - TString ToString() const; - void Out(IOutputStream& o) const; - + ui64 GetSizeBytes() const; + + void SerializeTo(Ydb::Table::TableIndex& proto) const; + TString ToString() const; + void Out(IOutputStream& o) const; + +private: + explicit TIndexDescription(const Ydb::Table::TableIndex& tableIndex); + explicit TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc); + + template <typename TProto> + static TIndexDescription FromProto(const TProto& proto); + private: - explicit TIndexDescription(const Ydb::Table::TableIndex& tableIndex); - explicit TIndexDescription(const Ydb::Table::TableIndexDescription& tableIndexDesc); - - template <typename TProto> - static TIndexDescription FromProto(const TProto& proto); - -private: TString IndexName_; - EIndexType IndexType_; + EIndexType IndexType_; TVector<TString> IndexColumns_; TVector<TString> DataColumns_; - ui64 SizeBytes = 0; + ui64 SizeBytes = 0; }; -bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs); -bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs); - +bool operator==(const TIndexDescription& lhs, const TIndexDescription& rhs); +bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs); + class TBuildIndexOperation : public TOperation { public: using TOperation::TOperation; @@ -196,117 +196,117 @@ struct TPartitionStats { ui64 Size = 0; }; -class TDateTypeColumnModeSettings { -public: - explicit TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter); - void SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const; - - const TString& GetColumnName() const; - const TDuration& GetExpireAfter() const; - -private: - TString ColumnName_; - TDuration ExpireAfter_; -}; - -class TValueSinceUnixEpochModeSettings { -public: - enum class EUnit { - Seconds, - MilliSeconds, - MicroSeconds, - NanoSeconds, - - Unknown = std::numeric_limits<int>::max() - }; - -public: - explicit TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); - void SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const; - - const TString& GetColumnName() const; - EUnit GetColumnUnit() const; - const TDuration& GetExpireAfter() const; - - static void Out(IOutputStream& o, EUnit unit); - static TString ToString(EUnit unit); - static EUnit UnitFromString(const TString& value); - -private: - TString ColumnName_; - EUnit ColumnUnit_; - TDuration ExpireAfter_; -}; - -//! Represents ttl settings -class TTtlSettings { -public: - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - - enum class EMode { - DateTypeColumn = 0, - ValueSinceUnixEpoch = 1, - }; - - explicit TTtlSettings(const TString& columnName, const TDuration& expireAfter); - explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds); - const TDateTypeColumnModeSettings& GetDateTypeColumn() const; - - explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); - explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds); - const TValueSinceUnixEpochModeSettings& GetValueSinceUnixEpoch() const; - - void SerializeTo(Ydb::Table::TtlSettings& proto) const; - EMode GetMode() const; - - TTtlSettings& SetRunInterval(const TDuration& value); - const TDuration& GetRunInterval() const; - -private: +class TDateTypeColumnModeSettings { +public: + explicit TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter); + void SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const; + + const TString& GetColumnName() const; + const TDuration& GetExpireAfter() const; + +private: + TString ColumnName_; + TDuration ExpireAfter_; +}; + +class TValueSinceUnixEpochModeSettings { +public: + enum class EUnit { + Seconds, + MilliSeconds, + MicroSeconds, + NanoSeconds, + + Unknown = std::numeric_limits<int>::max() + }; + +public: + explicit TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); + void SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const; + + const TString& GetColumnName() const; + EUnit GetColumnUnit() const; + const TDuration& GetExpireAfter() const; + + static void Out(IOutputStream& o, EUnit unit); + static TString ToString(EUnit unit); + static EUnit UnitFromString(const TString& value); + +private: + TString ColumnName_; + EUnit ColumnUnit_; + TDuration ExpireAfter_; +}; + +//! Represents ttl settings +class TTtlSettings { +public: + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + enum class EMode { + DateTypeColumn = 0, + ValueSinceUnixEpoch = 1, + }; + + explicit TTtlSettings(const TString& columnName, const TDuration& expireAfter); + explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds); + const TDateTypeColumnModeSettings& GetDateTypeColumn() const; + + explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); + explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds); + const TValueSinceUnixEpochModeSettings& GetValueSinceUnixEpoch() const; + + void SerializeTo(Ydb::Table::TtlSettings& proto) const; + EMode GetMode() const; + + TTtlSettings& SetRunInterval(const TDuration& value); + const TDuration& GetRunInterval() const; + +private: + std::variant< + TDateTypeColumnModeSettings, + TValueSinceUnixEpochModeSettings + > Mode_; + TDuration RunInterval_ = TDuration::Zero(); +}; + +class TAlterTtlSettings { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + + TAlterTtlSettings() + : Action_(true) + {} + + template <typename... Args> + explicit TAlterTtlSettings(Args&&... args) + : Action_(TTtlSettings(std::forward<Args>(args)...)) + {} + +public: + enum class EAction { + Drop = 0, + Set = 1, + }; + + static TAlterTtlSettings Drop() { + return TAlterTtlSettings(); + } + + template <typename... Args> + static TAlterTtlSettings Set(Args&&... args) { + return TAlterTtlSettings(std::forward<Args>(args)...); + } + + EAction GetAction() const; + const TTtlSettings& GetTtlSettings() const; + +private: std::variant< - TDateTypeColumnModeSettings, - TValueSinceUnixEpochModeSettings - > Mode_; - TDuration RunInterval_ = TDuration::Zero(); -}; - -class TAlterTtlSettings { - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - - TAlterTtlSettings() - : Action_(true) - {} - - template <typename... Args> - explicit TAlterTtlSettings(Args&&... args) - : Action_(TTtlSettings(std::forward<Args>(args)...)) - {} - -public: - enum class EAction { - Drop = 0, - Set = 1, - }; - - static TAlterTtlSettings Drop() { - return TAlterTtlSettings(); - } - - template <typename... Args> - static TAlterTtlSettings Set(Args&&... args) { - return TAlterTtlSettings(std::forward<Args>(args)...); - } - - EAction GetAction() const; - const TTtlSettings& GetTtlSettings() const; - -private: - std::variant< - bool, // EAction::Drop - TTtlSettings // EAction::Set - > Action_; -}; - + bool, // EAction::Drop + TTtlSettings // EAction::Set + > Action_; +}; + //! Represents table storage settings class TStorageSettings { public: @@ -387,8 +387,8 @@ class TTableDescription { friend class TTableBuilder; friend class NYdb::TProtoAccessor; - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + public: TTableDescription(Ydb::Table::DescribeTableResult&& desc, const TDescribeTableSettings& describeSettings); @@ -397,7 +397,7 @@ public: TVector<TColumn> GetColumns() const; TVector<TTableColumn> GetTableColumns() const; TVector<TIndexDescription> GetIndexDescriptions() const; - TMaybe<TTtlSettings> GetTtlSettings() const; + TMaybe<TTtlSettings> GetTtlSettings() const; const TString& GetOwner() const; const TVector<NScheme::TPermissions>& GetPermissions() const; @@ -428,9 +428,9 @@ public: // Returns column families of the table const TVector<TColumnFamilyDescription>& GetColumnFamilies() const; - // Attributes - const THashMap<TString, TString>& GetAttributes() const; - + // Attributes + const THashMap<TString, TString>& GetAttributes() const; + // Returns partitioning settings of the table const TPartitioningSettings& GetPartitioningSettings() const; @@ -440,37 +440,37 @@ public: // Returns read replicas settings of the table TMaybe<TReadReplicasSettings> GetReadReplicasSettings() const; - // Fills CreateTableRequest proto from this description - void SerializeTo(Ydb::Table::CreateTableRequest& request) const; - + // Fills CreateTableRequest proto from this description + void SerializeTo(Ydb::Table::CreateTableRequest& request) const; + private: TTableDescription(); - explicit TTableDescription(const Ydb::Table::CreateTableRequest& request); + explicit TTableDescription(const Ydb::Table::CreateTableRequest& request); void AddColumn(const TString& name, const Ydb::Type& type, const TString& family); void SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns); - - // common - void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns); - void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - // sync - void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - // async - void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - // default + + // common + void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns); + void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + // sync + void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + // async + void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + void AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + // default void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); void AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - - void SetTtlSettings(TTtlSettings&& settings); - void SetTtlSettings(const TTtlSettings& settings); - + + void SetTtlSettings(TTtlSettings&& settings); + void SetTtlSettings(const TTtlSettings& settings); + void SetStorageSettings(const TStorageSettings& settings); void AddColumnFamily(const TColumnFamilyDescription& desc); - void AddAttribute(const TString& key, const TString& value); - void SetAttributes(const THashMap<TString, TString>& attrs); - void SetAttributes(THashMap<TString, TString>&& attrs); + void AddAttribute(const TString& key, const TString& value); + void SetAttributes(const THashMap<TString, TString>& attrs); + void SetAttributes(THashMap<TString, TString>&& attrs); void SetCompactionPolicy(const TString& name); void SetUniformPartitions(ui64 partitionsCount); void SetPartitionAtKeys(const TExplicitPartitions& keys); @@ -478,8 +478,8 @@ private: void SetKeyBloomFilter(bool enabled); void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount); const Ydb::Table::DescribeTableResult& GetProto() const; - - class TImpl; + + class TImpl; std::shared_ptr<TImpl> Impl_; }; @@ -641,8 +641,8 @@ private: //////////////////////////////////////////////////////////////////////////////// class TTableBuilder { - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + public: TTableBuilder() = default; @@ -653,39 +653,39 @@ public: TTableBuilder& SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns); TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn); - // common - TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns); - TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn); - - // sync - TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TString& indexColumn); - - // async - TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TString& indexColumn); - - // default - TTableBuilder& AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); - TTableBuilder& AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); - TTableBuilder& AddSecondaryIndex(const TString& indexName, const TString& indexColumn); - - TTableBuilder& SetTtlSettings(TTtlSettings&& settings); - TTableBuilder& SetTtlSettings(const TTtlSettings& settings); - TTableBuilder& SetTtlSettings(const TString& columnName, const TDuration& expireAfter = TDuration::Zero()); - TTableBuilder& SetTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); - + // common + TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns); + TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn); + + // sync + TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + TTableBuilder& AddSyncSecondaryIndex(const TString& indexName, const TString& indexColumn); + + // async + TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + TTableBuilder& AddAsyncSecondaryIndex(const TString& indexName, const TString& indexColumn); + + // default + TTableBuilder& AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns); + TTableBuilder& AddSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns); + TTableBuilder& AddSecondaryIndex(const TString& indexName, const TString& indexColumn); + + TTableBuilder& SetTtlSettings(TTtlSettings&& settings); + TTableBuilder& SetTtlSettings(const TTtlSettings& settings); + TTableBuilder& SetTtlSettings(const TString& columnName, const TDuration& expireAfter = TDuration::Zero()); + TTableBuilder& SetTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); + TTableBuilder& SetStorageSettings(const TStorageSettings& settings); TTableBuilder& AddColumnFamily(const TColumnFamilyDescription& desc); - TTableBuilder& AddAttribute(const TString& key, const TString& value); - TTableBuilder& SetAttributes(const THashMap<TString, TString>& attrs); - TTableBuilder& SetAttributes(THashMap<TString, TString>&& attrs); - + TTableBuilder& AddAttribute(const TString& key, const TString& value); + TTableBuilder& SetAttributes(const THashMap<TString, TString>& attrs); + TTableBuilder& SetAttributes(THashMap<TString, TString>&& attrs); + TTableBuilder& SetCompactionPolicy(const TString& name); // UniformPartitions and PartitionAtKeys are mutually exclusive @@ -1036,23 +1036,23 @@ public: return TTxSettings(TS_STALE_RO); } - void Out(IOutputStream& out) const { - switch (Mode_) { - case TS_SERIALIZABLE_RW: - out << "SerializableRW"; - break; - case TS_ONLINE_RO: - out << "OnlineRO"; - break; - case TS_STALE_RO: - out << "StaleRO"; - break; - default: - out << "Unknown"; - break; - } - } - + void Out(IOutputStream& out) const { + switch (Mode_) { + case TS_SERIALIZABLE_RW: + out << "SerializableRW"; + break; + case TS_ONLINE_RO: + out << "OnlineRO"; + break; + case TS_STALE_RO: + out << "StaleRO"; + break; + default: + out << "Unknown"; + break; + } + } + private: enum ETransactionMode { TS_SERIALIZABLE_RW, @@ -1246,53 +1246,53 @@ private: TColumnFamilyBuilder Builder_; }; -class TAlterTtlSettingsBuilder { - using EUnit = TValueSinceUnixEpochModeSettings::EUnit; - -public: - TAlterTtlSettingsBuilder(TAlterTableSettings& parent); - - TAlterTtlSettingsBuilder& Drop(); - TAlterTtlSettingsBuilder& Set(TTtlSettings&& settings); - TAlterTtlSettingsBuilder& Set(const TTtlSettings& settings); - TAlterTtlSettingsBuilder& Set(const TString& columnName, const TDuration& expireAfter = TDuration::Zero()); - TAlterTtlSettingsBuilder& Set(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); - - TAlterTableSettings& EndAlterTtlSettings(); - -private: - TAlterTableSettings& Parent_; - - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -class TAlterAttributesBuilder { -public: - TAlterAttributesBuilder(TAlterTableSettings& parent) - : Parent_(parent) - { } - - TAlterAttributesBuilder& Alter(const TString& key, const TString& value) { - AlterAttributes_[key] = value; - return *this; - } - - TAlterAttributesBuilder& Add(const TString& key, const TString& value) { - return Alter(key, value); - } - - TAlterAttributesBuilder& Drop(const TString& key) { - return Alter(key, ""); - } - - TAlterTableSettings& EndAlterAttributes(); - -private: - TAlterTableSettings& Parent_; - THashMap<TString, TString> AlterAttributes_; -}; - +class TAlterTtlSettingsBuilder { + using EUnit = TValueSinceUnixEpochModeSettings::EUnit; + +public: + TAlterTtlSettingsBuilder(TAlterTableSettings& parent); + + TAlterTtlSettingsBuilder& Drop(); + TAlterTtlSettingsBuilder& Set(TTtlSettings&& settings); + TAlterTtlSettingsBuilder& Set(const TTtlSettings& settings); + TAlterTtlSettingsBuilder& Set(const TString& columnName, const TDuration& expireAfter = TDuration::Zero()); + TAlterTtlSettingsBuilder& Set(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter = TDuration::Zero()); + + TAlterTableSettings& EndAlterTtlSettings(); + +private: + TAlterTableSettings& Parent_; + + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +class TAlterAttributesBuilder { +public: + TAlterAttributesBuilder(TAlterTableSettings& parent) + : Parent_(parent) + { } + + TAlterAttributesBuilder& Alter(const TString& key, const TString& value) { + AlterAttributes_[key] = value; + return *this; + } + + TAlterAttributesBuilder& Add(const TString& key, const TString& value) { + return Alter(key, value); + } + + TAlterAttributesBuilder& Drop(const TString& key) { + return Alter(key, ""); + } + + TAlterTableSettings& EndAlterAttributes(); + +private: + TAlterTableSettings& Parent_; + THashMap<TString, TString> AlterAttributes_; +}; + class TAlterPartitioningSettingsBuilder { public: explicit TAlterPartitioningSettingsBuilder(TAlterTableSettings& parent) @@ -1333,10 +1333,10 @@ private: struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSettings> { using TSelf = TAlterTableSettings; - using TAlterAttributes = THashMap<TString, TString>; + using TAlterAttributes = THashMap<TString, TString>; + + TAlterTableSettings(); - TAlterTableSettings(); - FLUENT_SETTING_VECTOR(TTableColumn, AddColumns); FLUENT_SETTING_VECTOR(TString, DropColumns); @@ -1357,12 +1357,12 @@ struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSetting FLUENT_SETTING_VECTOR(TColumnFamilyDescription, AddColumnFamilies); FLUENT_SETTING_VECTOR(TColumnFamilyDescription, AlterColumnFamilies); - // workaround for MSVC - TSelf& AlterTtlSettings(const TMaybe<TAlterTtlSettings>& value); - const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const; - - FLUENT_SETTING(TAlterAttributes, AlterAttributes); - + // workaround for MSVC + TSelf& AlterTtlSettings(const TMaybe<TAlterTtlSettings>& value); + const TMaybe<TAlterTtlSettings>& GetAlterTtlSettings() const; + + FLUENT_SETTING(TAlterAttributes, AlterAttributes); + FLUENT_SETTING(TString, SetCompactionPolicy); FLUENT_SETTING_OPTIONAL(TPartitioningSettings, AlterPartitioningSettings); @@ -1386,22 +1386,22 @@ struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSetting TAlterColumnFamilyBuilder BeginAlterColumnFamily(const TString& name) { return TAlterColumnFamilyBuilder(*this, name); } - - TAlterTtlSettingsBuilder BeginAlterTtlSettings() { - return TAlterTtlSettingsBuilder(*this); - } - - TAlterAttributesBuilder BeginAlterAttributes() { - return TAlterAttributesBuilder(*this); - } + + TAlterTtlSettingsBuilder BeginAlterTtlSettings() { + return TAlterTtlSettingsBuilder(*this); + } + + TAlterAttributesBuilder BeginAlterAttributes() { + return TAlterAttributesBuilder(*this); + } TAlterPartitioningSettingsBuilder BeginAlterPartitioningSettings() { return TAlterPartitioningSettingsBuilder(*this); } - -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; }; inline TAlterTableSettings& TAlterStorageSettingsBuilder::EndAlterStorageSettings() { @@ -1416,10 +1416,10 @@ inline TAlterTableSettings& TAlterColumnFamilyBuilder::EndAlterColumnFamily() { return Parent_.AppendAlterColumnFamilies(Builder_.Build()); } -inline TAlterTableSettings& TAlterAttributesBuilder::EndAlterAttributes() { - return Parent_.AlterAttributes(AlterAttributes_); -} - +inline TAlterTableSettings& TAlterAttributesBuilder::EndAlterAttributes() { + return Parent_.AlterAttributes(AlterAttributes_); +} + inline TAlterTableSettings& TAlterPartitioningSettingsBuilder::EndAlterPartitioningSettings() { return Parent_.AlterPartitioningSettings(Builder_.Build()); } @@ -1868,15 +1868,15 @@ public: } // namespace NTable } // namespace NYdb - -Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TIndexDescription, o, x) { - return x.Out(o); -} - -Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TValueSinceUnixEpochModeSettings::EUnit, o, x) { - return NYdb::NTable::TValueSinceUnixEpochModeSettings::Out(o, x); -} - -Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TTxSettings, o, x) { - return x.Out(o); -} + +Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TIndexDescription, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TValueSinceUnixEpochModeSettings::EUnit, o, x) { + return NYdb::NTable::TValueSinceUnixEpochModeSettings::Out(o, x); +} + +Y_DECLARE_OUT_SPEC(inline, NYdb::NTable::TTxSettings, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h index 6be2c85ffd0..4a04b6ba5cd 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h @@ -1,7 +1,7 @@ #pragma once -#include <limits> - +#include <limits> + namespace NYdb { namespace NTable { @@ -22,12 +22,12 @@ enum class EBuildIndexState { Cancelled = 6, }; -enum class EIndexType { - GlobalSync, - GlobalAsync, - - Unknown = std::numeric_limits<int>::max() -}; - +enum class EIndexType { + GlobalSync, + GlobalAsync, + + Unknown = std::numeric_limits<int>::max() +}; + } // namespace NTable } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h index 1ea539f2119..bc5dce05ebc 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h @@ -48,6 +48,6 @@ using TAsyncOperation = NThreading::TFuture<TOperation>; } // namespace NYdb -Y_DECLARE_OUT_SPEC(inline, NYdb::TOperation, o, x) { +Y_DECLARE_OUT_SPEC(inline, NYdb::TOperation, o, x) { return x.Out(o); } diff --git a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h index 06e157e7c8f..fb8aa8ac685 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h @@ -17,15 +17,15 @@ struct TRequestSettings { FLUENT_SETTING(TString, TraceId); FLUENT_SETTING(TString, RequestType); FLUENT_SETTING(THeader, Header); - - TRequestSettings() = default; - - template <typename T> - explicit TRequestSettings(const TRequestSettings<T>& other) - : TraceId_(other.TraceId_) - , RequestType_(other.RequestType_) + + TRequestSettings() = default; + + template <typename T> + explicit TRequestSettings(const TRequestSettings<T>& other) + : TraceId_(other.TraceId_) + , RequestType_(other.RequestType_) , Header_(other.Header_) - {} + {} }; template<typename TDerived> @@ -33,14 +33,14 @@ struct TSimpleRequestSettings : public TRequestSettings<TDerived> { using TSelf = TDerived; FLUENT_SETTING(TDuration, ClientTimeout); - - TSimpleRequestSettings() = default; - - template <typename T> - explicit TSimpleRequestSettings(const TSimpleRequestSettings<T>& other) - : TRequestSettings<TDerived>(other) - , ClientTimeout_(other.ClientTimeout_) - {} + + TSimpleRequestSettings() = default; + + template <typename T> + explicit TSimpleRequestSettings(const TSimpleRequestSettings<T>& other) + : TRequestSettings<TDerived>(other) + , ClientTimeout_(other.ClientTimeout_) + {} }; template<typename TDerived> @@ -53,17 +53,17 @@ struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); - TOperationRequestSettings() = default; - - template <typename T> - explicit TOperationRequestSettings(const TOperationRequestSettings<T>& other) - : TSimpleRequestSettings<TDerived>(other) - , OperationTimeout_(other.OperationTimeout_) - , CancelAfter_(other.CancelAfter_) - , UseClientTimeoutForOperation_(other.UseClientTimeoutForOperation_) + TOperationRequestSettings() = default; + + template <typename T> + explicit TOperationRequestSettings(const TOperationRequestSettings<T>& other) + : TSimpleRequestSettings<TDerived>(other) + , OperationTimeout_(other.OperationTimeout_) + , CancelAfter_(other.CancelAfter_) + , UseClientTimeoutForOperation_(other.UseClientTimeoutForOperation_) , ReportCostInfo_(other.ReportCostInfo_) - {} - + {} + TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) { CancelAfter_ = cancelAfter; OperationTimeout_ = operationTimeout; diff --git a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h index fc8a1ed2f92..dca95fdb7ad 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h @@ -1,23 +1,23 @@ -#pragma once - -#include "fluent_settings_helpers.h" - -namespace NYdb { - -enum class ES3Scheme { - HTTP = 1 /* "http" */, - HTTPS = 2 /* "https" */, -}; - -template <typename TDerived> -struct TS3Settings { - using TSelf = TDerived; - - FLUENT_SETTING(TString, Endpoint); - FLUENT_SETTING_DEFAULT(ES3Scheme, Scheme, ES3Scheme::HTTPS); - FLUENT_SETTING(TString, Bucket); - FLUENT_SETTING(TString, AccessKey); - FLUENT_SETTING(TString, SecretKey); -}; - -} // namespace NYdb +#pragma once + +#include "fluent_settings_helpers.h" + +namespace NYdb { + +enum class ES3Scheme { + HTTP = 1 /* "http" */, + HTTPS = 2 /* "https" */, +}; + +template <typename TDerived> +struct TS3Settings { + using TSelf = TDerived; + + FLUENT_SETTING(TString, Endpoint); + FLUENT_SETTING_DEFAULT(ES3Scheme, Scheme, ES3Scheme::HTTPS); + FLUENT_SETTING(TString, Bucket); + FLUENT_SETTING(TString, AccessKey); + FLUENT_SETTING(TString, SecretKey); +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/ya.make b/ydb/public/sdk/cpp/client/ydb_types/ya.make index d5c2a921f5f..d1419ecd34b 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/ya.make @@ -8,7 +8,7 @@ PEERDIR( ydb/library/yql/public/issue ) -GENERATE_ENUM_SERIALIZATION(s3_settings.h) +GENERATE_ENUM_SERIALIZATION(s3_settings.h) GENERATE_ENUM_SERIALIZATION(status_codes.h) END() diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index af5f370a389..8c18d4d50f8 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -91,14 +91,14 @@ TType::TType(const Ydb::Type& typeProto) TType::TType(Ydb::Type&& typeProto) : Impl_(new TImpl(std::move(typeProto))) {} -TString TType::ToString() const { - return FormatType(*this); -} - -void TType::Out(IOutputStream& o) const { - o << FormatType(*this); -} - +TString TType::ToString() const { + return FormatType(*this); +} + +void TType::Out(IOutputStream& o) const { + o << FormatType(*this); +} + const Ydb::Type& TType::GetProto() const { return Impl_->ProtoType_; } diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index f87c802cae3..4a163b92206 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -21,9 +21,9 @@ public: TType(const Ydb::Type& typeProto); TType(Ydb::Type&& typeProto); - TString ToString() const; - void Out(IOutputStream& o) const; - + TString ToString() const; + void Out(IOutputStream& o) const; + private: const Ydb::Type& GetProto() const; @@ -453,7 +453,7 @@ public: }; } // namespace NYdb - -Y_DECLARE_OUT_SPEC(inline, NYdb::TType, o, x) { - return x.Out(o); -} + +Y_DECLARE_OUT_SPEC(inline, NYdb::TType, o, x) { + return x.Out(o); +} diff --git a/ydb/public/sdk/python/CHANGELOG.md b/ydb/public/sdk/python/CHANGELOG.md index 070aa61c8eb..139ce6f71fc 100644 --- a/ydb/public/sdk/python/CHANGELOG.md +++ b/ydb/public/sdk/python/CHANGELOG.md @@ -71,10 +71,10 @@ * change error format for sqlalchemy -## 1.0.12 ## - -* add ``ValueSinceUnixEpochModeSettings`` support to ``TtlSettings`` - +## 1.0.12 ## + +* add ``ValueSinceUnixEpochModeSettings`` support to ``TtlSettings`` + ## 1.0.11 ## * add default credentials constructor diff --git a/ydb/public/sdk/python/ydb/_session_impl.py b/ydb/public/sdk/python/ydb/_session_impl.py index 88ca250b781..e1906299ecd 100644 --- a/ydb/public/sdk/python/ydb/_session_impl.py +++ b/ydb/public/sdk/python/ydb/_session_impl.py @@ -1,5 +1,5 @@ import functools -from google.protobuf.empty_pb2 import Empty +from google.protobuf.empty_pb2 import Empty from . import issues, types, _apis, convert, scheme, operation, _utilities X_YDB_SERVER_HINTS = "x-ydb-server-hints" @@ -180,11 +180,11 @@ def create_table_request_factory(session_state, path, table_description): for index in table_description.indexes: request.indexes.add().MergeFrom(index.to_pb()) - if table_description.ttl_settings is not None: + if table_description.ttl_settings is not None: request.ttl_settings.MergeFrom(table_description.ttl_settings.to_pb()) - - request.attributes.update(table_description.attributes) - + + request.attributes.update(table_description.attributes) + if table_description.column_families: for column_family in table_description.column_families: request.column_families.add().MergeFrom(column_family.to_pb()) @@ -277,7 +277,7 @@ def describe_table_request_factory(session_state, path, settings=None): return request -def alter_table_request_factory( +def alter_table_request_factory( session_state, path, add_columns, @@ -310,15 +310,15 @@ def alter_table_request_factory( for index in add_indexes: request.add_indexes.add().MergeFrom(index.to_pb()) - if alter_attributes is not None: - request.alter_attributes.update(alter_attributes) + if alter_attributes is not None: + request.alter_attributes.update(alter_attributes) + + if set_ttl_settings is not None: + request.set_ttl_settings.MergeFrom(set_ttl_settings.to_pb()) + + if drop_ttl_settings is not None and drop_ttl_settings: + request.drop_ttl_settings.MergeFrom(Empty()) - if set_ttl_settings is not None: - request.set_ttl_settings.MergeFrom(set_ttl_settings.to_pb()) - - if drop_ttl_settings is not None and drop_ttl_settings: - request.drop_ttl_settings.MergeFrom(Empty()) - if add_column_families is not None: for column_family in add_column_families: request.add_column_families.add().MergeFrom(column_family.to_pb()) diff --git a/ydb/public/sdk/python/ydb/connection.py b/ydb/public/sdk/python/ydb/connection.py index 214c1743c31..573b276e207 100644 --- a/ydb/public/sdk/python/ydb/connection.py +++ b/ydb/public/sdk/python/ydb/connection.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) DEFAULT_TIMEOUT = 600 YDB_DATABASE_HEADER = "x-ydb-database" YDB_TRACE_ID_HEADER = "x-ydb-trace-id" -YDB_REQUEST_TYPE_HEADER = "x-ydb-request-type" +YDB_REQUEST_TYPE_HEADER = "x-ydb-request-type" def _message_to_string(message): @@ -141,12 +141,12 @@ def _construct_metadata(driver_config, settings): if driver_config.credentials is not None: metadata.extend(driver_config.credentials.auth_metadata()) - if settings is not None: - if settings.trace_id is not None: - metadata.append((YDB_TRACE_ID_HEADER, settings.trace_id)) - if settings.request_type is not None: - metadata.append((YDB_REQUEST_TYPE_HEADER, settings.request_type)) - + if settings is not None: + if settings.trace_id is not None: + metadata.append((YDB_TRACE_ID_HEADER, settings.trace_id)) + if settings.request_type is not None: + metadata.append((YDB_REQUEST_TYPE_HEADER, settings.request_type)) + metadata.append(_utilities.x_ydb_sdk_build_info_header()) return metadata diff --git a/ydb/public/sdk/python/ydb/scheme.py b/ydb/public/sdk/python/ydb/scheme.py index cf513cffb2c..b5f1a7f7e5f 100644 --- a/ydb/public/sdk/python/ydb/scheme.py +++ b/ydb/public/sdk/python/ydb/scheme.py @@ -20,7 +20,7 @@ class SchemeEntryType(enum.IntEnum): BLOCK_STORE_VOLUME = 6 COORDINATION_NODE = 7 SEQUENCE = 15 - REPLICATION = 16 + REPLICATION = 16 @staticmethod def is_table(entry): @@ -91,14 +91,14 @@ class SchemeEntry(object): :param type: A type of scheme entry :param effective_permissions: A list of effective permissions applied to this scheme entry :param permissions: A list of permissions applied to this scheme entry - :param size_bytes: Size of entry in bytes + :param size_bytes: Size of entry in bytes """ self.name = name self.owner = owner self.type = type self.effective_permissions = effective_permissions self.permissions = permissions - self.size_bytes = size_bytes + self.size_bytes = size_bytes def is_directory(self): """ @@ -300,7 +300,7 @@ def _wrap_scheme_entry(entry_pb, scheme_entry_cls=None, *args, **kwargs): getattr(SchemeEntryType, _apis.ydb_scheme.Entry.Type.Name(entry_pb.type)), _wrap_permissions(entry_pb.effective_permissions), _wrap_permissions(entry_pb.permissions), - entry_pb.size_bytes, + entry_pb.size_bytes, *args, **kwargs ) diff --git a/ydb/public/sdk/python/ydb/settings.py b/ydb/public/sdk/python/ydb/settings.py index 890ffd6a2ee..96094c4978e 100644 --- a/ydb/public/sdk/python/ydb/settings.py +++ b/ydb/public/sdk/python/ydb/settings.py @@ -16,7 +16,7 @@ class BaseRequestSettings(object): Request settings to be used for RPC execution """ self.trace_id = None - self.request_type = None + self.request_type = None self.timeout = None self.cancel_after = None self.operation_timeout = None @@ -30,15 +30,15 @@ class BaseRequestSettings(object): self.trace_id = trace_id return self - def with_request_type(self, request_type): - """ - Includes request type for RPC headers - :param request_type: A request type string - :return: The self instance - """ - self.request_type = request_type - return self - + def with_request_type(self, request_type): + """ + Includes request type for RPC headers + :param request_type: A request type string + :return: The self instance + """ + self.request_type = request_type + return self + def with_operation_timeout(self, timeout): """ Indicates that client is no longer interested in the result of operation after the specified duration diff --git a/ydb/public/sdk/python/ydb/table.py b/ydb/public/sdk/python/ydb/table.py index 34a33f41d33..c49c4660c7c 100644 --- a/ydb/public/sdk/python/ydb/table.py +++ b/ydb/public/sdk/python/ydb/table.py @@ -452,77 +452,77 @@ class TableProfile(object): return pb -class DateTypeColumnModeSettings(object): - def __init__(self, column_name, expire_after_seconds=0): - self.column_name = column_name - self.expire_after_seconds = expire_after_seconds - - def to_pb(self): - pb = _apis.ydb_table.DateTypeColumnModeSettings() - - pb.column_name = self.column_name - pb.expire_after_seconds = self.expire_after_seconds - - return pb - - -@enum.unique -class ColumnUnit(enum.IntEnum): - UNIT_UNSPECIFIED = 0 - UNIT_SECONDS = 1 - UNIT_MILLISECONDS = 2 - UNIT_MICROSECONDS = 3 - UNIT_NANOSECONDS = 4 - - -class ValueSinceUnixEpochModeSettings(object): - def __init__(self, column_name, column_unit, expire_after_seconds=0): - self.column_name = column_name - self.column_unit = column_unit - self.expire_after_seconds = expire_after_seconds - - def to_pb(self): - pb = _apis.ydb_table.ValueSinceUnixEpochModeSettings() - - pb.column_name = self.column_name - pb.column_unit = self.column_unit - pb.expire_after_seconds = self.expire_after_seconds - - return pb - - -class TtlSettings(object): - def __init__(self): - self.date_type_column = None - self.value_since_unix_epoch = None - - def with_date_type_column(self, column_name, expire_after_seconds=0): +class DateTypeColumnModeSettings(object): + def __init__(self, column_name, expire_after_seconds=0): + self.column_name = column_name + self.expire_after_seconds = expire_after_seconds + + def to_pb(self): + pb = _apis.ydb_table.DateTypeColumnModeSettings() + + pb.column_name = self.column_name + pb.expire_after_seconds = self.expire_after_seconds + + return pb + + +@enum.unique +class ColumnUnit(enum.IntEnum): + UNIT_UNSPECIFIED = 0 + UNIT_SECONDS = 1 + UNIT_MILLISECONDS = 2 + UNIT_MICROSECONDS = 3 + UNIT_NANOSECONDS = 4 + + +class ValueSinceUnixEpochModeSettings(object): + def __init__(self, column_name, column_unit, expire_after_seconds=0): + self.column_name = column_name + self.column_unit = column_unit + self.expire_after_seconds = expire_after_seconds + + def to_pb(self): + pb = _apis.ydb_table.ValueSinceUnixEpochModeSettings() + + pb.column_name = self.column_name + pb.column_unit = self.column_unit + pb.expire_after_seconds = self.expire_after_seconds + + return pb + + +class TtlSettings(object): + def __init__(self): + self.date_type_column = None + self.value_since_unix_epoch = None + + def with_date_type_column(self, column_name, expire_after_seconds=0): self.date_type_column = DateTypeColumnModeSettings( column_name, expire_after_seconds ) - return self - + return self + def with_value_since_unix_epoch( self, column_name, column_unit, expire_after_seconds=0 ): self.value_since_unix_epoch = ValueSinceUnixEpochModeSettings( column_name, column_unit, expire_after_seconds ) - return self - - def to_pb(self): - pb = _apis.ydb_table.TtlSettings() - - if self.date_type_column is not None: - pb.date_type_column.MergeFrom(self.date_type_column.to_pb()) - elif self.value_since_unix_epoch is not None: - pb.value_since_unix_epoch.MergeFrom(self.value_since_unix_epoch.to_pb()) - else: - raise RuntimeError("Unspecified ttl settings mode") - - return pb - - + return self + + def to_pb(self): + pb = _apis.ydb_table.TtlSettings() + + if self.date_type_column is not None: + pb.date_type_column.MergeFrom(self.date_type_column.to_pb()) + elif self.value_since_unix_epoch is not None: + pb.value_since_unix_epoch.MergeFrom(self.value_since_unix_epoch.to_pb()) + else: + raise RuntimeError("Unspecified ttl settings mode") + + return pb + + class TableStats(object): def __init__(self): self.partitions = None @@ -680,8 +680,8 @@ class TableDescription(object): self.profile = None self.indexes = [] self.column_families = [] - self.ttl_settings = None - self.attributes = {} + self.ttl_settings = None + self.attributes = {} self.uniform_partitions = 0 self.partition_at_keys = None self.compaction_policy = None @@ -734,18 +734,18 @@ class TableDescription(object): self.profile = profile return self - def with_ttl(self, ttl_settings): - self.ttl_settings = ttl_settings - return self + def with_ttl(self, ttl_settings): + self.ttl_settings = ttl_settings + return self + + def with_attributes(self, attributes): + self.attributes = attributes + return self - def with_attributes(self, attributes): - self.attributes = attributes - return self - def with_uniform_partitions(self, uniform_partitions): self.uniform_partitions = uniform_partitions return self - + def with_partition_at_keys(self, partition_at_keys): self.partition_at_keys = partition_at_keys return self @@ -1460,19 +1460,19 @@ class TableSchemeEntry(scheme.SchemeEntry): getattr(partitioning_settings, field), ) - self.ttl_settings = None - if ttl_settings is not None: + self.ttl_settings = None + if ttl_settings is not None: if ttl_settings.HasField("date_type_column"): - self.ttl_settings = TtlSettings().with_date_type_column( - ttl_settings.date_type_column.column_name, + self.ttl_settings = TtlSettings().with_date_type_column( + ttl_settings.date_type_column.column_name, ttl_settings.date_type_column.expire_after_seconds, - ) + ) elif ttl_settings.HasField("value_since_unix_epoch"): - self.ttl_settings = TtlSettings().with_value_since_unix_epoch( - ttl_settings.value_since_unix_epoch.column_name, - ColumnUnit(ttl_settings.value_since_unix_epoch.column_unit), + self.ttl_settings = TtlSettings().with_value_since_unix_epoch( + ttl_settings.value_since_unix_epoch.column_name, + ColumnUnit(ttl_settings.value_since_unix_epoch.column_unit), ttl_settings.value_since_unix_epoch.expire_after_seconds, - ) + ) self.table_stats = None if table_stats is not None: @@ -1487,9 +1487,9 @@ class TableSchemeEntry(scheme.SchemeEntry): table_stats.store_size ) - self.attributes = attributes - - + self.attributes = attributes + + class RenameItem: def __init__(self, source_path, destination_path, replace_destination=False): self._source_path = source_path @@ -1726,12 +1726,12 @@ class BaseSession(ISession): set_read_replicas_settings=None, ): return self._driver( - _session_impl.alter_table_request_factory( + _session_impl.alter_table_request_factory( self._state, path, add_columns, drop_columns, - alter_attributes, + alter_attributes, add_indexes, drop_indexes, set_ttl_settings, @@ -1743,7 +1743,7 @@ class BaseSession(ISession): alter_partitioning_settings, set_key_bloom_filter, set_read_replicas_settings, - ), + ), _apis.TableService.Stub, _apis.TableService.AlterTable, _session_impl.AlterTableOperation, @@ -1959,12 +1959,12 @@ class Session(BaseSession): set_read_replicas_settings=None, ): return self._driver.future( - _session_impl.alter_table_request_factory( + _session_impl.alter_table_request_factory( self._state, path, add_columns, drop_columns, - alter_attributes, + alter_attributes, add_indexes, drop_indexes, set_ttl_settings, @@ -1976,7 +1976,7 @@ class Session(BaseSession): alter_partitioning_settings, set_key_bloom_filter, set_read_replicas_settings, - ), + ), _apis.TableService.Stub, _apis.TableService.AlterTable, _session_impl.AlterTableOperation, diff --git a/ydb/public/tools/lib/cmds/__init__.py b/ydb/public/tools/lib/cmds/__init__.py index f37e7d5602a..977f0a4aae6 100644 --- a/ydb/public/tools/lib/cmds/__init__.py +++ b/ydb/public/tools/lib/cmds/__init__.py @@ -26,7 +26,7 @@ class EmptyArguments(object): self.ydb_udfs_dir = None self.yql_config_path = None self.auth_config_path = None - self.debug_logging = [] + self.debug_logging = [] self.fixed_ports = False @@ -71,22 +71,22 @@ def wrap_path(path): ) -def write_file_flushed(filename, content): - with open(filename, 'w') as fd: - fd.write(content) - fd.flush() - os.fsync(fd.fileno()) - - +def write_file_flushed(filename, content): + with open(filename, 'w') as fd: + fd.write(content) + fd.flush() + os.fsync(fd.fileno()) + + def write_file(args, suffix, content): if args.ydb_working_dir: - write_file_flushed(os.path.join(args.ydb_working_dir, suffix), content) + write_file_flushed(os.path.join(args.ydb_working_dir, suffix), content) return - write_file_flushed(os.path.join(yatest_common.output_path(suffix)), content) + write_file_flushed(os.path.join(yatest_common.output_path(suffix)), content) try: - write_file_flushed(suffix, content) + write_file_flushed(suffix, content) except IOError: return @@ -256,14 +256,14 @@ def deploy(arguments): arguments.ydb_binary_path = driver_path_packages(arguments.use_packages) arguments.ydb_udfs_dir = None - additional_log_configs = {} + additional_log_configs = {} if getattr(arguments, 'debug_logging', []): debug_logging = getattr(arguments, 'debug_logging', []) - additional_log_configs = { - k: LogLevels.DEBUG + additional_log_configs = { + k: LogLevels.DEBUG for k in debug_logging - } - + } + port_allocator = None if getattr(arguments, 'fixed_ports', False): port_allocator = KikimrFixedPortAllocator([KikimrFixedNodePortAllocator()]) @@ -286,7 +286,7 @@ def deploy(arguments): load_udfs=True, suppress_version_check=arguments.suppress_version_check, udfs_path=arguments.ydb_udfs_dir, - additional_log_configs=additional_log_configs, + additional_log_configs=additional_log_configs, port_allocator=port_allocator, use_in_memory_pdisks=use_in_memory_pdisks_flag(), yql_config_path=arguments.yql_config_path, diff --git a/ydb/services/datastreams/grpc_service.cpp b/ydb/services/datastreams/grpc_service.cpp index 76f040eb8c3..7a18cd4dc67 100644 --- a/ydb/services/datastreams/grpc_service.cpp +++ b/ydb/services/datastreams/grpc_service.cpp @@ -47,10 +47,10 @@ void TGRpcDataStreamsService::DecRequest() { void TGRpcDataStreamsService::InitNewSchemeCache() { auto appData = ActorSystem_->AppData<TAppData>(); - auto cacheCounters = GetServiceCounters(Counters_, "pqproxy|schemecache"); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); - NewSchemeCache = ActorSystem_->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), - TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); + auto cacheCounters = GetServiceCounters(Counters_, "pqproxy|schemecache"); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); + NewSchemeCache = ActorSystem_->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), + TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId); } void TGRpcDataStreamsService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h index badb03a035f..65c8aca1db2 100644 --- a/ydb/services/persqueue_v1/grpc_pq_actor.h +++ b/ydb/services/persqueue_v1/grpc_pq_actor.h @@ -488,10 +488,10 @@ private: HFunc(TEvDescribeTopicsResponse, Handle) - HFunc(NPQ::TEvPartitionWriter::TEvInitResult, Handle); - HFunc(NPQ::TEvPartitionWriter::TEvWriteAccepted, Handle); - HFunc(NPQ::TEvPartitionWriter::TEvWriteResponse, Handle); - HFunc(NPQ::TEvPartitionWriter::TEvDisconnected, Handle); + HFunc(NPQ::TEvPartitionWriter::TEvInitResult, Handle); + HFunc(NPQ::TEvPartitionWriter::TEvWriteAccepted, Handle); + HFunc(NPQ::TEvPartitionWriter::TEvWriteResponse, Handle); + HFunc(NPQ::TEvPartitionWriter::TEvDisconnected, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); @@ -533,10 +533,10 @@ private: THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx); void InitCheckACL(const TActorContext& ctx); - void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx); - void Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx); - void Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx); - void Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr& ev, const TActorContext& ctx); + void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx); + void Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx); + void Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx); + void Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr& ev, const TActorContext& ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPersQueue::TEvGetPartitionIdForWriteResponse::TPtr& ev, const NActors::TActorContext& ctx); @@ -564,15 +564,15 @@ private: ES_WAIT_NEXT_PARTITION = 5, ES_WAIT_TABLE_REQUEST_2 = 6, ES_WAIT_TABLE_REQUEST_3 = 7, - ES_WAIT_WRITER_INIT = 8, - ES_INITED = 9 + ES_WAIT_WRITER_INIT = 8, + ES_INITED = 9 //TODO: filter }; EState State; TActorId SchemeCache; TActorId NewSchemeCache; - TActorId Writer; + TActorId Writer; TString PeerName; ui64 Cookie; @@ -599,10 +599,10 @@ private: // Source requests from user (grpc session object) std::deque<THolder<TEvPQProxy::TEvWrite>> UserWriteRequests; - // Formed write request's size + // Formed write request's size ui64 ByteSize = 0; - // Formed write request's cookie + // Formed write request's cookie ui64 Cookie = 0; }; diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index 7661084acb4..654edfcfcd4 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -1928,11 +1928,11 @@ bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, cons TString sourceId; if (!r.GetSourceId().empty()) { - if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { + if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_PROXY, "read bad sourceId from " << Partition << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "'"); } - sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); + sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); } if (!currentBatch || currentBatch->write_timestamp_ms() != r.GetWriteTimestampMS() || currentBatch->source_id() != sourceId) { diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index d5052a8e920..3038cc82a61 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -225,8 +225,8 @@ void TWriteSessionActor::Handle(IContext::TEvWriteFinished::TPtr& ev, const TAct void TWriteSessionActor::Die(const TActorContext& ctx) { - if (Writer) - ctx.Send(Writer, new TEvents::TEvPoisonPill()); + if (Writer) + ctx.Send(Writer, new TEvents::TEvPoisonPill()); if (SessionsActive) { SessionsActive.Dec(); @@ -318,7 +318,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor SourceId = init.message_group_id(); TString encodedSourceId; try { - encodedSourceId = NPQ::NSourceIdEncoding::Encode(SourceId); + encodedSourceId = NPQ::NSourceIdEncoding::Encode(SourceId); } catch (yexception& e) { CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), PersQueue::ErrorCode::BAD_REQUEST, ctx); return; @@ -698,8 +698,8 @@ void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorCont return; } - Writer = ctx.RegisterWithSameMailbox(NPQ::CreatePartitionWriter(ctx.SelfID, tabletId, Partition, SourceId)); - State = ES_WAIT_WRITER_INIT; + Writer = ctx.RegisterWithSameMailbox(NPQ::CreatePartitionWriter(ctx.SelfID, tabletId, Partition, SourceId)); + State = ES_WAIT_WRITER_INIT; ui32 border = AppData(ctx)->PQConfig.GetWriteInitLatencyBigMs(); auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); @@ -746,83 +746,83 @@ void TWriteSessionActor::CloseSession(const TString& errorReason, const PersQueu Die(ctx); } -void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx) { - if (State != ES_WAIT_WRITER_INIT) { - return CloseSession("got init result but not wait for it", PersQueue::ErrorCode::ERROR, ctx); - } +void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx) { + if (State != ES_WAIT_WRITER_INIT) { + return CloseSession("got init result but not wait for it", PersQueue::ErrorCode::ERROR, ctx); + } - const auto& result = *ev->Get(); - if (!result.IsSuccess()) { - const auto& error = result.GetError(); - if (error.Response.HasErrorCode()) { + const auto& result = *ev->Get(); + if (!result.IsSuccess()) { + const auto& error = result.GetError(); + if (error.Response.HasErrorCode()) { return CloseSession("status is not ok: " + error.Response.GetErrorReason(), ConvertOldCode(error.Response.GetErrorCode()), ctx); } else { - return CloseSession("error at writer init: " + error.Reason, PersQueue::ErrorCode::ERROR, ctx); + return CloseSession("error at writer init: " + error.Reason, PersQueue::ErrorCode::ERROR, ctx); } } - OwnerCookie = result.GetResult().OwnerCookie; - const auto& maxSeqNo = result.GetResult().SourceIdInfo.GetSeqNo(); + OwnerCookie = result.GetResult().OwnerCookie; + const auto& maxSeqNo = result.GetResult().SourceIdInfo.GetSeqNo(); - StreamingWriteServerMessage response; - response.set_status(Ydb::StatusIds::SUCCESS); - auto init = response.mutable_init_response(); - init->set_session_id(EscapeC(OwnerCookie)); - init->set_last_sequence_number(maxSeqNo); - init->set_partition_id(Partition); + StreamingWriteServerMessage response; + response.set_status(Ydb::StatusIds::SUCCESS); + auto init = response.mutable_init_response(); + init->set_session_id(EscapeC(OwnerCookie)); + init->set_last_sequence_number(maxSeqNo); + init->set_partition_id(Partition); init->set_topic(TopicConverter->GetModernName()); init->set_cluster(TopicConverter->GetCluster()); - init->set_block_format_version(0); - if (InitialPQTabletConfig.HasCodecs()) { - for (const auto& codecName : InitialPQTabletConfig.GetCodecs().GetCodecs()) { - init->add_supported_codecs(CodecByName(codecName)); - } - } - - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition - << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie); - - if (!Request->GetStreamCtx()->Write(std::move(response))) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc write failed"); - Die(ctx); - return; - } - - State = ES_INITED; - - ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup()); - - //init completed; wait for first data chunk - NextRequestInited = true; - if (!Request->GetStreamCtx()->Read()) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read failed"); - Die(ctx); + init->set_block_format_version(0); + if (InitialPQTabletConfig.HasCodecs()) { + for (const auto& codecName : InitialPQTabletConfig.GetCodecs().GetCodecs()) { + init->add_supported_codecs(CodecByName(codecName)); + } + } + + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition + << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie); + + if (!Request->GetStreamCtx()->Write(std::move(response))) { + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc write failed"); + Die(ctx); return; } -} -void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx) { - if (State != ES_INITED) { - return CloseSession("got write permission but not wait for it", PersQueue::ErrorCode::ERROR, ctx); - } + State = ES_INITED; - Y_VERIFY(!FormedWrites.empty()); - TWriteRequestBatchInfo::TPtr writeRequest = std::move(FormedWrites.front()); + ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup()); + + //init completed; wait for first data chunk + NextRequestInited = true; + if (!Request->GetStreamCtx()->Read()) { + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read failed"); + Die(ctx); + return; + } +} + +void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx) { + if (State != ES_INITED) { + return CloseSession("got write permission but not wait for it", PersQueue::ErrorCode::ERROR, ctx); + } + + Y_VERIFY(!FormedWrites.empty()); + TWriteRequestBatchInfo::TPtr writeRequest = std::move(FormedWrites.front()); if (ev->Get()->Cookie != writeRequest->Cookie) { return CloseSession("out of order reserve bytes response from server, may be previous is lost", PersQueue::ErrorCode::ERROR, ctx); - } + } - FormedWrites.pop_front(); + FormedWrites.pop_front(); - ui64 diff = writeRequest->ByteSize; + ui64 diff = writeRequest->ByteSize; - SentMessages.emplace_back(std::move(writeRequest)); + SentMessages.emplace_back(std::move(writeRequest)); - BytesInflight_ -= diff; - BytesInflight.Dec(diff); + BytesInflight_ -= diff; + BytesInflight.Dec(diff); - if (!NextRequestInited && BytesInflight_ < MAX_BYTES_INFLIGHT) { //allow only one big request to be readed but not sended + if (!NextRequestInited && BytesInflight_ < MAX_BYTES_INFLIGHT) { //allow only one big request to be readed but not sended NextRequestInited = true; if (!Request->GetStreamCtx()->Read()) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read failed"); @@ -831,97 +831,97 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& } } - --NumReserveBytesRequests; - if (!Writes.empty()) - GenerateNextWriteRequest(ctx); -} + --NumReserveBytesRequests; + if (!Writes.empty()) + GenerateNextWriteRequest(ctx); +} -void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx) { - if (State != ES_INITED) { - return CloseSession("got write response but not wait for it", PersQueue::ErrorCode::ERROR, ctx); - } +void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx) { + if (State != ES_INITED) { + return CloseSession("got write response but not wait for it", PersQueue::ErrorCode::ERROR, ctx); + } - const auto& result = *ev->Get(); - if (!result.IsSuccess()) { - const auto& record = result.Record; - if (record.HasErrorCode()) { + const auto& result = *ev->Get(); + if (!result.IsSuccess()) { + const auto& record = result.Record; + if (record.HasErrorCode()) { return CloseSession("status is not ok: " + record.GetErrorReason(), ConvertOldCode(record.GetErrorCode()), ctx); - } else { - return CloseSession("error at write: " + result.GetError().Reason, PersQueue::ErrorCode::ERROR, ctx); - } - } + } else { + return CloseSession("error at write: " + result.GetError().Reason, PersQueue::ErrorCode::ERROR, ctx); + } + } - const auto& resp = result.Record.GetPartitionResponse(); + const auto& resp = result.Record.GetPartitionResponse(); - if (SentMessages.empty()) { - CloseSession("got too many replies from server, internal error", PersQueue::ErrorCode::ERROR, ctx); - return; - } + if (SentMessages.empty()) { + CloseSession("got too many replies from server, internal error", PersQueue::ErrorCode::ERROR, ctx); + return; + } - TWriteRequestBatchInfo::TPtr writeRequest = std::move(SentMessages.front()); - SentMessages.pop_front(); + TWriteRequestBatchInfo::TPtr writeRequest = std::move(SentMessages.front()); + SentMessages.pop_front(); if (resp.GetCookie() != writeRequest->Cookie) { return CloseSession("out of order write response from server, may be previous is lost", PersQueue::ErrorCode::ERROR, ctx); - } - - auto addAck = [](const TPersQueuePartitionResponse::TCmdWriteResult& res, StreamingWriteServerMessage::BatchWriteResponse* batchWriteResponse, - StreamingWriteServerMessage::WriteStatistics* stat) { - batchWriteResponse->add_sequence_numbers(res.GetSeqNo()); - batchWriteResponse->add_offsets(res.GetOffset()); - batchWriteResponse->add_already_written(res.GetAlreadyWritten()); - - stat->set_queued_in_partition_duration_ms( - Max((i64)res.GetTotalTimeInPartitionQueueMs(), stat->queued_in_partition_duration_ms())); - stat->set_throttled_on_partition_duration_ms( - Max((i64)res.GetPartitionQuotedTimeMs(), stat->throttled_on_partition_duration_ms())); - stat->set_throttled_on_topic_duration_ms(Max(static_cast<i64>(res.GetTopicQuotedTimeMs()), stat->throttled_on_topic_duration_ms())); - stat->set_persist_duration_ms( - Max((i64)res.GetWriteTimeMs(), stat->persist_duration_ms())); - }; - - ui32 partitionCmdWriteResultIndex = 0; - // TODO: Send single batch write response for all user write requests up to some max size/count - for (const auto& userWriteRequest : writeRequest->UserWriteRequests) { - StreamingWriteServerMessage result; - result.set_status(Ydb::StatusIds::SUCCESS); - auto batchWriteResponse = result.mutable_batch_write_response(); - batchWriteResponse->set_partition_id(Partition); - - for (size_t messageIndex = 0, endIndex = userWriteRequest->Request.write_request().sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) { - if (partitionCmdWriteResultIndex == resp.CmdWriteResultSize()) { - CloseSession("too less responses from server", PersQueue::ErrorCode::ERROR, ctx); - return; + } + + auto addAck = [](const TPersQueuePartitionResponse::TCmdWriteResult& res, StreamingWriteServerMessage::BatchWriteResponse* batchWriteResponse, + StreamingWriteServerMessage::WriteStatistics* stat) { + batchWriteResponse->add_sequence_numbers(res.GetSeqNo()); + batchWriteResponse->add_offsets(res.GetOffset()); + batchWriteResponse->add_already_written(res.GetAlreadyWritten()); + + stat->set_queued_in_partition_duration_ms( + Max((i64)res.GetTotalTimeInPartitionQueueMs(), stat->queued_in_partition_duration_ms())); + stat->set_throttled_on_partition_duration_ms( + Max((i64)res.GetPartitionQuotedTimeMs(), stat->throttled_on_partition_duration_ms())); + stat->set_throttled_on_topic_duration_ms(Max(static_cast<i64>(res.GetTopicQuotedTimeMs()), stat->throttled_on_topic_duration_ms())); + stat->set_persist_duration_ms( + Max((i64)res.GetWriteTimeMs(), stat->persist_duration_ms())); + }; + + ui32 partitionCmdWriteResultIndex = 0; + // TODO: Send single batch write response for all user write requests up to some max size/count + for (const auto& userWriteRequest : writeRequest->UserWriteRequests) { + StreamingWriteServerMessage result; + result.set_status(Ydb::StatusIds::SUCCESS); + auto batchWriteResponse = result.mutable_batch_write_response(); + batchWriteResponse->set_partition_id(Partition); + + for (size_t messageIndex = 0, endIndex = userWriteRequest->Request.write_request().sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) { + if (partitionCmdWriteResultIndex == resp.CmdWriteResultSize()) { + CloseSession("too less responses from server", PersQueue::ErrorCode::ERROR, ctx); + return; } - const auto& partitionCmdWriteResult = resp.GetCmdWriteResult(partitionCmdWriteResultIndex); - const auto writtenSequenceNumber = userWriteRequest->Request.write_request().sequence_numbers(messageIndex); - if (partitionCmdWriteResult.GetSeqNo() != writtenSequenceNumber) { - CloseSession(TStringBuilder() << "Expected partition " << Partition << " write result for message with sequence number " << writtenSequenceNumber << " but got for " << partitionCmdWriteResult.GetSeqNo(), PersQueue::ErrorCode::ERROR, ctx); + const auto& partitionCmdWriteResult = resp.GetCmdWriteResult(partitionCmdWriteResultIndex); + const auto writtenSequenceNumber = userWriteRequest->Request.write_request().sequence_numbers(messageIndex); + if (partitionCmdWriteResult.GetSeqNo() != writtenSequenceNumber) { + CloseSession(TStringBuilder() << "Expected partition " << Partition << " write result for message with sequence number " << writtenSequenceNumber << " but got for " << partitionCmdWriteResult.GetSeqNo(), PersQueue::ErrorCode::ERROR, ctx); return; } - addAck(partitionCmdWriteResult, batchWriteResponse, batchWriteResponse->mutable_write_statistics()); - ++partitionCmdWriteResultIndex; - } + addAck(partitionCmdWriteResult, batchWriteResponse, batchWriteResponse->mutable_write_statistics()); + ++partitionCmdWriteResultIndex; + } - if (!Request->GetStreamCtx()->Write(std::move(result))) { - // TODO: Log gRPC write error code - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc write failed"); - Die(ctx); - return; - } - } + if (!Request->GetStreamCtx()->Write(std::move(result))) { + // TODO: Log gRPC write error code + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc write failed"); + Die(ctx); + return; + } + } - ui64 diff = writeRequest->ByteSize; + ui64 diff = writeRequest->ByteSize; - BytesInflightTotal_ -= diff; - BytesInflightTotal.Dec(diff); + BytesInflightTotal_ -= diff; + BytesInflightTotal.Dec(diff); - CheckFinish(ctx); -} + CheckFinish(ctx); +} -void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr&, const TActorContext& ctx) { - CloseSession("pipe to partition's tablet is dead", PersQueue::ErrorCode::ERROR, ctx); +void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr&, const TActorContext& ctx) { + CloseSession("pipe to partition's tablet is dead", PersQueue::ErrorCode::ERROR, ctx); } void TWriteSessionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { @@ -941,9 +941,9 @@ void TWriteSessionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, con void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) { TWriteRequestBatchInfo::TPtr writeRequest = new TWriteRequestBatchInfo(); - auto ev = MakeHolder<NPQ::TEvPartitionWriter::TEvWriteRequest>(++NextRequestCookie); - NKikimrClient::TPersQueueRequest& request = ev->Record; - + auto ev = MakeHolder<NPQ::TEvPartitionWriter::TEvWriteRequest>(++NextRequestCookie); + NKikimrClient::TPersQueueRequest& request = ev->Record; + writeRequest->UserWriteRequests = std::move(Writes); Writes.clear(); @@ -952,7 +952,7 @@ void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) { auto w = request.MutablePartitionRequest()->AddCmdWrite(); w->SetData(GetSerializedData(InitMeta, writeRequest, messageIndex)); w->SetSeqNo(writeRequest.sequence_numbers(messageIndex)); - w->SetSourceId(NPQ::NSourceIdEncoding::EncodeSimple(SourceId)); + w->SetSourceId(NPQ::NSourceIdEncoding::EncodeSimple(SourceId)); w->SetCreateTimeMS(writeRequest.created_at_ms(messageIndex)); w->SetUncompressedSize(writeRequest.blocks_uncompressed_sizes(messageIndex)); w->SetClientDC(ClientDC); @@ -966,7 +966,7 @@ void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) { } } - writeRequest->Cookie = request.GetPartitionRequest().GetCookie(); + writeRequest->Cookie = request.GetPartitionRequest().GetCookie(); Y_VERIFY(-diff <= (i64)BytesInflight_); diff += request.ByteSize(); @@ -978,7 +978,7 @@ void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) { writeRequest->ByteSize = request.ByteSize(); FormedWrites.push_back(writeRequest); - ctx.Send(Writer, std::move(ev)); + ctx.Send(Writer, std::move(ev)); ++NumReserveBytesRequests; } diff --git a/ydb/services/persqueue_v1/persqueue.cpp b/ydb/services/persqueue_v1/persqueue.cpp index b3363c9e18c..ccacc4d43af 100644 --- a/ydb/services/persqueue_v1/persqueue.cpp +++ b/ydb/services/persqueue_v1/persqueue.cpp @@ -60,10 +60,10 @@ void TGRpcPersQueueService::DecRequest() { void TGRpcPersQueueService::InitNewSchemeCacheActor() { auto appData = ActorSystem->AppData<TAppData>(); - auto cacheCounters = GetServiceCounters(Counters, "pqproxy|schemecache"); - auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); - NewSchemeCache = ActorSystem->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), - TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); + auto cacheCounters = GetServiceCounters(Counters, "pqproxy|schemecache"); + auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); + NewSchemeCache = ActorSystem->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()), + TMailboxType::HTSwap, ActorSystem->AppData<TAppData>()->UserPoolId); } void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { diff --git a/ydb/services/persqueue_v1/persqueue_utils.cpp b/ydb/services/persqueue_v1/persqueue_utils.cpp index b638e6b68d6..51d7f38045d 100644 --- a/ydb/services/persqueue_v1/persqueue_utils.cpp +++ b/ydb/services/persqueue_v1/persqueue_utils.cpp @@ -1,7 +1,7 @@ #include "persqueue_utils.h" #include <ydb/core/base/path.h> - + namespace NKikimr::NGRpcProxy::V1 { TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl) diff --git a/ydb/services/ydb/ya.make b/ydb/services/ydb/ya.make index bc6cf1f724f..a2f41a3e75d 100644 --- a/ydb/services/ydb/ya.make +++ b/ydb/services/ydb/ya.make @@ -9,10 +9,10 @@ SRCS( ydb_clickhouse_internal.cpp ydb_dummy.cpp ydb_experimental.cpp - ydb_export.cpp - ydb_import.cpp + ydb_export.cpp + ydb_import.cpp ydb_logstore.cpp - ydb_operation.cpp + ydb_operation.cpp ydb_s3_internal.cpp ydb_scheme.cpp ydb_scripting.cpp diff --git a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp index 221e658ab5c..0665008ee49 100644 --- a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp +++ b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp @@ -926,10 +926,10 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { } } - void Index(NYdb::NTable::EIndexType indexType, bool enableBulkUpsertToAsyncIndexedTables = false) { - auto server = TKikimrWithGrpcAndRootSchema({}, {}, {}, false, nullptr, [=](auto& settings) { - settings.SetEnableBulkUpsertToAsyncIndexedTables(enableBulkUpsertToAsyncIndexedTables); - }); + void Index(NYdb::NTable::EIndexType indexType, bool enableBulkUpsertToAsyncIndexedTables = false) { + auto server = TKikimrWithGrpcAndRootSchema({}, {}, {}, false, nullptr, [=](auto& settings) { + settings.SetEnableBulkUpsertToAsyncIndexedTables(enableBulkUpsertToAsyncIndexedTables); + }); ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; @@ -944,7 +944,7 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { tableBuilder .AddNullableColumn("Key", EPrimitiveType::Uint8) .AddNullableColumn("Value", EPrimitiveType::Uint8) - .AddSecondaryIndex("Value_index", indexType, "Value"); + .AddSecondaryIndex("Value_index", indexType, "Value"); tableBuilder.SetPrimaryKeyColumns({"Key"}); auto result = session.CreateTable("/Root/ui8", tableBuilder.Build()).ExtractValueSync(); @@ -956,24 +956,24 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { { auto res = TestUpsertRow(client, "/Root/ui8", 1, 2); - - if (indexType == NYdb::NTable::EIndexType::GlobalAsync) { - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), enableBulkUpsertToAsyncIndexedTables - ? EStatus::SUCCESS : EStatus::SCHEME_ERROR); - - while (enableBulkUpsertToAsyncIndexedTables) { - auto it = session.ReadTable("/Root/ui8/Value_index/indexImplTable").ExtractValueSync(); - auto streamPart = it.ReadNext().GetValueSync(); - auto str = NYdb::FormatResultSetYson(streamPart.ExtractPart()); - if (str == "[[[2u];[1u]]]") { - break; - } - - Sleep(TDuration::Seconds(1)); - } - } else { - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); - } + + if (indexType == NYdb::NTable::EIndexType::GlobalAsync) { + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), enableBulkUpsertToAsyncIndexedTables + ? EStatus::SUCCESS : EStatus::SCHEME_ERROR); + + while (enableBulkUpsertToAsyncIndexedTables) { + auto it = session.ReadTable("/Root/ui8/Value_index/indexImplTable").ExtractValueSync(); + auto streamPart = it.ReadNext().GetValueSync(); + auto str = NYdb::FormatResultSetYson(streamPart.ExtractPart()); + if (str == "[[[2u];[1u]]]") { + break; + } + + Sleep(TDuration::Seconds(1)); + } + } else { + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); + } } { @@ -982,18 +982,18 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { } } - Y_UNIT_TEST(SyncIndexShouldSucceed) { - Index(NYdb::NTable::EIndexType::GlobalSync); - } - - Y_UNIT_TEST(AsyncIndexShouldFail) { - Index(NYdb::NTable::EIndexType::GlobalAsync, false); - } - - Y_UNIT_TEST(AsyncIndexShouldSucceed) { - Index(NYdb::NTable::EIndexType::GlobalAsync, true); - } - + Y_UNIT_TEST(SyncIndexShouldSucceed) { + Index(NYdb::NTable::EIndexType::GlobalSync); + } + + Y_UNIT_TEST(AsyncIndexShouldFail) { + Index(NYdb::NTable::EIndexType::GlobalAsync, false); + } + + Y_UNIT_TEST(AsyncIndexShouldSucceed) { + Index(NYdb::NTable::EIndexType::GlobalAsync, true); + } + Y_UNIT_TEST(Timeout) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index d9711746c01..c067928c785 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -70,13 +70,13 @@ public: } ServerSettings->SetAppConfig(appConfig); ServerSettings->AuthConfig = appConfig.GetAuthConfig(); - ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); + ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); ServerSettings->SetKqpSettings(kqpSettings); - ServerSettings->SetEnableAsyncIndexes(true); + ServerSettings->SetEnableAsyncIndexes(true); ServerSettings->SetEnableDataColumnForIndexTable(true); ServerSettings->SetEnableNotNullColumns(true); - ServerSettings->SetEnableSystemViews(TestSettings::EnableSystemViews); - ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); + ServerSettings->SetEnableSystemViews(TestSettings::EnableSystemViews); + ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); ServerSettings->SetEnableYq(enableYq); ServerSettings->Formats = new TFormatFactory; ServerSettings->PQConfig = appConfig.GetPQConfig(); diff --git a/ydb/services/ydb/ydb_export.cpp b/ydb/services/ydb/ydb_export.cpp index 6fb158887a3..7a0933ce555 100644 --- a/ydb/services/ydb/ydb_export.cpp +++ b/ydb/services/ydb/ydb_export.cpp @@ -1,58 +1,58 @@ -#include "ydb_export.h" - +#include "ydb_export.h" + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + TGRpcYdbExportService::TGRpcYdbExportService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ } - + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ } + void TGRpcYdbExportService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; + CQ_ = cq; SetupIncomingRequests(std::move(logger)); -} - +} + void TGRpcYdbExportService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - -bool TGRpcYdbExportService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcYdbExportService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - + Limiter_ = limiter; +} + +bool TGRpcYdbExportService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcYdbExportService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + void TGRpcYdbExportService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::Export::IN, Ydb::Export::OUT, TGRpcYdbExportService>>(this, &Service_, CQ_, \ + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + MakeIntrusive<TGRpcRequest<Ydb::Export::IN, Ydb::Export::OUT, TGRpcYdbExportService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ACTION; \ - }, &Ydb::Export::V1::ExportService::AsyncService::Request ## NAME, \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ACTION; \ + }, &Ydb::Export::V1::ExportService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("export", #NAME))->Run(); - - ADD_REQUEST(ExportToYt, ExportToYtRequest, ExportToYtResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvExportToYtRequest(ctx)); - }) - ADD_REQUEST(ExportToS3, ExportToS3Request, ExportToS3Response, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvExportToS3Request(ctx)); - }) -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr + + ADD_REQUEST(ExportToYt, ExportToYtRequest, ExportToYtResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvExportToYtRequest(ctx)); + }) + ADD_REQUEST(ExportToS3, ExportToS3Request, ExportToS3Response, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvExportToS3Request(ctx)); + }) +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_export.h b/ydb/services/ydb/ydb_export.h index c0dedf49023..6049462ca22 100644 --- a/ydb/services/ydb/ydb_export.h +++ b/ydb/services/ydb/ydb_export.h @@ -1,33 +1,33 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/ydb_export_v1.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -class TGRpcYdbExportService + +namespace NKikimr { +namespace NGRpcService { + +class TGRpcYdbExportService : public NGrpc::TGrpcServiceBase<Ydb::Export::V1::ExportService> -{ -public: +{ +public: TGRpcYdbExportService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); - + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: + + bool IncRequest(); + void DecRequest(); +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_import.cpp b/ydb/services/ydb/ydb_import.cpp index cb0f457a34c..c27578329d4 100644 --- a/ydb/services/ydb/ydb_import.cpp +++ b/ydb/services/ydb/ydb_import.cpp @@ -1,58 +1,58 @@ -#include "ydb_import.h" - +#include "ydb_import.h" + #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_calls.h> - -namespace NKikimr { -namespace NGRpcService { - -TGRpcYdbImportService::TGRpcYdbImportService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) -{ } - + +namespace NKikimr { +namespace NGRpcService { + +TGRpcYdbImportService::TGRpcYdbImportService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) +{ } + void TGRpcYdbImportService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; + CQ_ = cq; SetupIncomingRequests(std::move(logger)); -} - +} + void TGRpcYdbImportService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - -bool TGRpcYdbImportService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcYdbImportService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - + Limiter_ = limiter; +} + +bool TGRpcYdbImportService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcYdbImportService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + void TGRpcYdbImportService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::Import::IN, Ydb::Import::OUT, TGRpcYdbImportService>>(this, &Service_, CQ_, \ + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + MakeIntrusive<TGRpcRequest<Ydb::Import::IN, Ydb::Import::OUT, TGRpcYdbImportService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ - NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ - ACTION; \ - }, &Ydb::Import::V1::ImportService::AsyncService::Request ## NAME, \ + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ + ACTION; \ + }, &Ydb::Import::V1::ImportService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("import", #NAME))->Run(); - - ADD_REQUEST(ImportFromS3, ImportFromS3Request, ImportFromS3Response, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvImportFromS3Request(ctx)); - }) - ADD_REQUEST(ImportData, ImportDataRequest, ImportDataResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvImportDataRequest(ctx)); - }) -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr + + ADD_REQUEST(ImportFromS3, ImportFromS3Request, ImportFromS3Response, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvImportFromS3Request(ctx)); + }) + ADD_REQUEST(ImportData, ImportDataRequest, ImportDataResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvImportDataRequest(ctx)); + }) +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_import.h b/ydb/services/ydb/ydb_import.h index cb137b76b97..a7db52853a5 100644 --- a/ydb/services/ydb/ydb_import.h +++ b/ydb/services/ydb/ydb_import.h @@ -1,33 +1,33 @@ -#pragma once - -#include <library/cpp/actors/core/actorsystem.h> +#pragma once + +#include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/ydb_import_v1.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -class TGRpcYdbImportService + +namespace NKikimr { +namespace NGRpcService { + +class TGRpcYdbImportService : public NGrpc::TGrpcServiceBase<Ydb::Import::V1::ImportService> -{ -public: - TGRpcYdbImportService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); - +{ +public: + TGRpcYdbImportService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id); + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: + + bool IncRequest(); + void DecRequest(); +private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; - NActors::TActorId GRpcRequestProxyId_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/ydb/ydb_logstore_ut.cpp b/ydb/services/ydb/ydb_logstore_ut.cpp index 9cd057ebbb8..fa7679f1a2b 100644 --- a/ydb/services/ydb/ydb_logstore_ut.cpp +++ b/ydb/services/ydb/ydb_logstore_ut.cpp @@ -350,7 +350,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Create table with TTL settings { - NYdb::NLogStore::TTtlSettings ttlSettings("saved_at", TDuration::Seconds(2000)); + NYdb::NLogStore::TTtlSettings ttlSettings("saved_at", TDuration::Seconds(2000)); NYdb::NLogStore::TLogTableDescription tableDescr("default", {"timestamp", "uid"}, 4, ttlSettings); auto res = logStoreClient.CreateLogTable("/Root/LogStore/log2", std::move(tableDescr)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); @@ -368,7 +368,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Add TTL to a table (currently not supported) { NYdb::NLogStore::TAlterLogTableSettings alterLogTableSettings; - alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("uint_timestamp", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600))); + alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("uint_timestamp", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600))); auto res = logStoreClient.AlterLogTable("/Root/LogStore/log1", std::move(alterLogTableSettings)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::GENERIC_ERROR, res.GetIssues().ToString()); } @@ -383,7 +383,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Change TTL column (currently not supported) { NYdb::NLogStore::TAlterLogTableSettings alterLogTableSettings; - alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("ingested_at", TDuration::Seconds(86400))); + alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("ingested_at", TDuration::Seconds(86400))); auto res = logStoreClient.AlterLogTable("/Root/LogStore/log2", std::move(alterLogTableSettings)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::GENERIC_ERROR, res.GetIssues().ToString()); } @@ -400,7 +400,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Change TTL expiration time { NYdb::NLogStore::TAlterLogTableSettings alterLogTableSettings; - alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("saved_at", TDuration::Seconds(86400))); + alterLogTableSettings.AlterTtlSettings(NYdb::NTable::TAlterTtlSettings::Set("saved_at", TDuration::Seconds(86400))); auto res = logStoreClient.AlterLogTable("/Root/LogStore/log2", std::move(alterLogTableSettings)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); } @@ -433,7 +433,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Use invalid column for TTL { - NYdb::NLogStore::TTtlSettings ttlSettings("nonexisting_column", TDuration::Seconds(2000)); + NYdb::NLogStore::TTtlSettings ttlSettings("nonexisting_column", TDuration::Seconds(2000)); NYdb::NLogStore::TLogTableDescription tableDescr("default", {"timestamp", "uid"}, 4, ttlSettings); auto res = logStoreClient.CreateLogTable("/Root/LogStore/log3", std::move(tableDescr)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::GENERIC_ERROR, res.GetIssues().ToString()); @@ -441,7 +441,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Use column of invalid type for TTL { - NYdb::NLogStore::TTtlSettings ttlSettings("message", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600)); + NYdb::NLogStore::TTtlSettings ttlSettings("message", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600)); NYdb::NLogStore::TLogTableDescription tableDescr("default", {"timestamp", "uid"}, 4, ttlSettings); auto res = logStoreClient.CreateLogTable("/Root/LogStore/log4", std::move(tableDescr)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::GENERIC_ERROR, res.GetIssues().ToString()); @@ -449,7 +449,7 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { // Use non-Timestamp column for TTL { - NYdb::NLogStore::TTtlSettings ttlSettings("uint_timestamp", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600)); + NYdb::NLogStore::TTtlSettings ttlSettings("uint_timestamp", NYdb::NTable::TTtlSettings::EUnit::MilliSeconds, TDuration::Seconds(3600)); NYdb::NLogStore::TLogTableDescription tableDescr("default", {"timestamp", "uid"}, 4, ttlSettings); auto res = logStoreClient.CreateLogTable("/Root/LogStore/log5", std::move(tableDescr)).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::GENERIC_ERROR, res.GetIssues().ToString()); diff --git a/ydb/services/ydb/ydb_operation.cpp b/ydb/services/ydb/ydb_operation.cpp index ed30c97b02f..602980027cb 100644 --- a/ydb/services/ydb/ydb_operation.cpp +++ b/ydb/services/ydb/ydb_operation.cpp @@ -1,4 +1,4 @@ -#include "ydb_operation.h" +#include "ydb_operation.h" #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/grpc_request_proxy.h> @@ -22,11 +22,11 @@ void TGRpcOperationService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter *limite Limiter_ = limiter; } -bool TGRpcOperationService::IncRequest() { +bool TGRpcOperationService::IncRequest() { return Limiter_->Inc(); } -void TGRpcOperationService::DecRequest() { +void TGRpcOperationService::DecRequest() { Limiter_->Dec(); Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); } @@ -37,7 +37,7 @@ void TGRpcOperationService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { #error ADD_REQUEST macro already defined #endif #define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - MakeIntrusive<TGRpcRequest<Ydb::Operations::IN, Ydb::Operations::OUT, TGRpcOperationService>>(this, &Service_, CQ_, \ + MakeIntrusive<TGRpcRequest<Ydb::Operations::IN, Ydb::Operations::OUT, TGRpcOperationService>>(this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ACTION; \ @@ -47,15 +47,15 @@ void TGRpcOperationService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { ADD_REQUEST(GetOperation, GetOperationRequest, GetOperationResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvGetOperationRequest(ctx)); }) - ADD_REQUEST(CancelOperation, CancelOperationRequest, CancelOperationResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvCancelOperationRequest(ctx)); - }) - ADD_REQUEST(ForgetOperation, ForgetOperationRequest, ForgetOperationResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvForgetOperationRequest(ctx)); - }) - ADD_REQUEST(ListOperations, ListOperationsRequest, ListOperationsResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvListOperationsRequest(ctx)); - }) + ADD_REQUEST(CancelOperation, CancelOperationRequest, CancelOperationResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvCancelOperationRequest(ctx)); + }) + ADD_REQUEST(ForgetOperation, ForgetOperationRequest, ForgetOperationResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvForgetOperationRequest(ctx)); + }) + ADD_REQUEST(ListOperations, ListOperationsRequest, ListOperationsResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvListOperationsRequest(ctx)); + }) #undef ADD_REQUEST } diff --git a/ydb/services/ydb/ydb_operation.h b/ydb/services/ydb/ydb_operation.h index adc6c292ee1..9f7e8646b1d 100644 --- a/ydb/services/ydb/ydb_operation.h +++ b/ydb/services/ydb/ydb_operation.h @@ -7,7 +7,7 @@ namespace NKikimr { namespace NGRpcService { -class TGRpcOperationService +class TGRpcOperationService : public NGrpc::TGrpcServiceBase<Ydb::Operation::V1::OperationService> { public: diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index 8fda10d45f7..4861c06b92e 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -3061,7 +3061,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 , {"/Root/Table-2", "/Root/Table-4"}} ).ExtractValueSync(); UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetStatus()); // do not fail on exist + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetStatus()); // do not fail on exist } { diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index 2697aac1e4c..2d370630799 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -4437,7 +4437,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { points.add_split_points()->CopyFrom(point); keyVal.mutable_items(0)->set_uint64_value(20); points.add_split_points()->CopyFrom(point); - CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::SCHEME_ERROR); + CreateTable(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::SCHEME_ERROR); } Y_UNIT_TEST(ExplicitPartitionsComplex) { @@ -4512,7 +4512,7 @@ Y_UNIT_TEST_SUITE(TTableProfileTests) { keyVal.add_items()->set_bytes_value("value1"); auto &points = *policy.mutable_explicit_partitions(); points.add_split_points()->CopyFrom(point); - CreateTableComplexKey(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::SCHEME_ERROR); + CreateTableComplexKey(server, "/Root/ydb_ut_tenant/table-1", profile, Ydb::StatusIds::SCHEME_ERROR); } void CheckRepeated(const ::google::protobuf::RepeatedPtrField<TString> &array, diff --git a/ydb/tests/functional/api/test_public_api.py b/ydb/tests/functional/api/test_public_api.py index c788a3cba3f..072ee35de38 100644 --- a/ydb/tests/functional/api/test_public_api.py +++ b/ydb/tests/functional/api/test_public_api.py @@ -3,10 +3,10 @@ import codecs import decimal import time import json -import os +import os import pytest -from hamcrest import any_of, assert_that, equal_to, is_, none, raises, less_than, has_length, not_ +from hamcrest import any_of, assert_that, equal_to, is_, none, raises, less_than, has_length, not_ from tornado import ioloop, gen from google.protobuf import text_format import logging @@ -115,28 +115,28 @@ class Base(object): cls.driver.stop() -class WithTenant(Base): - @classmethod - def setup_class(cls): +class WithTenant(Base): + @classmethod + def setup_class(cls): cls.cluster = kikimr_cluster_factory() - cls.cluster.start() - cls.database_name = "/Root/tenant" - cls.cluster.create_database( - cls.database_name, + cls.cluster.start() + cls.database_name = "/Root/tenant" + cls.cluster.create_database( + cls.database_name, storage_pool_units_count={'hdd': 1} - ) + ) cls.cluster.register_and_start_slots(cls.database_name, count=1) cls.cluster.wait_tenant_up(cls.database_name) cls.connection_params = ydb.DriverConfig(endpoint="localhost:%d" % cls.cluster.nodes[1].port, database=cls.database_name) - cls.driver = ydb.Driver(cls.connection_params) - cls.driver.wait() - - @classmethod - def teardown_class(cls): - cls.cluster.remove_database(cls.database_name) - super(WithTenant, cls).teardown_class() - - + cls.driver = ydb.Driver(cls.connection_params) + cls.driver.wait() + + @classmethod + def teardown_class(cls): + cls.cluster.remove_database(cls.database_name) + super(WithTenant, cls).teardown_class() + + class TestExplain(Base): def test_explain_data_query(self): session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) @@ -1763,219 +1763,219 @@ class TestForPotentialDeadlock(object): connection = driver._store.get() driver.stop() connection.close() - - -class TestRecursiveCreation(Base): - def test_mkdir(self): - path = '{}/test_recursive_mkdir/a/b/dir'.format(self.database_name) - - self.driver.scheme_client.make_directory(path) - response = self.driver.scheme_client.describe_path(path) - assert_that(response.name, is_(os.path.basename(path))) - - def test_create_table(self): - path = '{}/test_recursive_create_table/a/b/table'.format(self.database_name) - - self.driver.table_client.session().create().create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - ) - response = self.driver.scheme_client.describe_path(path) - assert_that(response.name, is_(os.path.basename(path))) - - -class TestAttributes(WithTenant): - ATTRIBUTES = { - 'one': '1', - 'two': 'two', - } - - def test_create_table(self): - path = '{}/test_attributes/create_table/table'.format(self.database_name) + + +class TestRecursiveCreation(Base): + def test_mkdir(self): + path = '{}/test_recursive_mkdir/a/b/dir'.format(self.database_name) + + self.driver.scheme_client.make_directory(path) + response = self.driver.scheme_client.describe_path(path) + assert_that(response.name, is_(os.path.basename(path))) + + def test_create_table(self): + path = '{}/test_recursive_create_table/a/b/table'.format(self.database_name) + + self.driver.table_client.session().create().create_table( + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + ) + response = self.driver.scheme_client.describe_path(path) + assert_that(response.name, is_(os.path.basename(path))) + + +class TestAttributes(WithTenant): + ATTRIBUTES = { + 'one': '1', + 'two': 'two', + } + + def test_create_table(self): + path = '{}/test_attributes/create_table/table'.format(self.database_name) session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) session.create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - .with_attributes(self.ATTRIBUTES) - ) - + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + .with_attributes(self.ATTRIBUTES) + ) + response = session.describe_table(path) - assert_that(response.attributes, equal_to(self.ATTRIBUTES)) - - def test_copy_table(self): - src_path = '{}/test_attributes/copy_table/src_table'.format(self.database_name) - dst_path = '{}/test_attributes/copy_table/dst_table'.format(self.database_name) - + assert_that(response.attributes, equal_to(self.ATTRIBUTES)) + + def test_copy_table(self): + src_path = '{}/test_attributes/copy_table/src_table'.format(self.database_name) + dst_path = '{}/test_attributes/copy_table/dst_table'.format(self.database_name) + session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) session.create_table( - src_path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - .with_attributes(self.ATTRIBUTES) - ) - + src_path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + .with_attributes(self.ATTRIBUTES) + ) + session.copy_table(src_path, dst_path) - + response = session.describe_table(dst_path) - assert_that(response.attributes, equal_to(self.ATTRIBUTES)) - - def test_create_indexed_table(self): - path = '{}/test_attributes/create_indexed_table/table'.format(self.database_name) - + assert_that(response.attributes, equal_to(self.ATTRIBUTES)) + + def test_create_indexed_table(self): + path = '{}/test_attributes/create_indexed_table/table'.format(self.database_name) + session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) session.create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - .with_index( - ydb.TableIndex('by_value').with_index_columns('value') - ) - .with_attributes(self.ATTRIBUTES) - ) - + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + .with_index( + ydb.TableIndex('by_value').with_index_columns('value') + ) + .with_attributes(self.ATTRIBUTES) + ) + response = session.describe_table(path) - assert_that(response.attributes, equal_to(self.ATTRIBUTES)) - - def test_alter_table(self): - path = '{}/test_attributes/alter_table/table'.format(self.database_name) - - # create + assert_that(response.attributes, equal_to(self.ATTRIBUTES)) + + def test_alter_table(self): + path = '{}/test_attributes/alter_table/table'.format(self.database_name) + + # create session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) session.create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - ) - + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + ) + response = session.describe_table(path) - assert_that(response.attributes, equal_to({})) - - # alter (add) + assert_that(response.attributes, equal_to({})) + + # alter (add) session.alter_table( - path, add_columns=[], drop_columns=[], alter_attributes=self.ATTRIBUTES - ) - + path, add_columns=[], drop_columns=[], alter_attributes=self.ATTRIBUTES + ) + response = session.describe_table(path) - assert_that(response.attributes, equal_to(self.ATTRIBUTES)) - - # alter (drop) + assert_that(response.attributes, equal_to(self.ATTRIBUTES)) + + # alter (drop) session.alter_table( - path, add_columns=[], drop_columns=[], alter_attributes={ - k: '' for k in self.ATTRIBUTES.keys() - } - ) - + path, add_columns=[], drop_columns=[], alter_attributes={ + k: '' for k in self.ATTRIBUTES.keys() + } + ) + response = session.describe_table(path) - assert_that(response.attributes, equal_to({})) - - # mixed alter - def mixed_alter(): + assert_that(response.attributes, equal_to({})) + + # mixed alter + def mixed_alter(): session.alter_table( - path, add_columns=[], drop_columns=['value'], alter_attributes=self.ATTRIBUTES - ) - - assert_that(mixed_alter, raises(ydb.Unsupported)) - - @pytest.mark.parametrize('attributes', [ - {'': 'v'}, # empty key - {'k': ''}, # empty value - {'k' * (100 + 1): 'v'}, # too long key - {'k': 'v' * (4096 + 1)}, # too long value - {'k1': 'v1' * 2048, 'k2': 'v2' * 2048, 'k3': 'v3' * 2048}, # bad total size - ]) - def test_limits(self, attributes): - path = '{}/test_attributes/limits/table'.format(self.database_name) + path, add_columns=[], drop_columns=['value'], alter_attributes=self.ATTRIBUTES + ) + + assert_that(mixed_alter, raises(ydb.Unsupported)) + + @pytest.mark.parametrize('attributes', [ + {'': 'v'}, # empty key + {'k': ''}, # empty value + {'k' * (100 + 1): 'v'}, # too long key + {'k': 'v' * (4096 + 1)}, # too long value + {'k1': 'v1' * 2048, 'k2': 'v2' * 2048, 'k3': 'v3' * 2048}, # bad total size + ]) + def test_limits(self, attributes): + path = '{}/test_attributes/limits/table'.format(self.database_name) session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) - - def callee(): + + def callee(): session.create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') - .with_attributes(attributes) - ) - - assert_that(callee, any_of(raises(ydb.BadRequest), raises(ydb.GenericError))) - - -class TestDocApiTables(WithTenant): - def _create_table(self, path): + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') + .with_attributes(attributes) + ) + + assert_that(callee, any_of(raises(ydb.BadRequest), raises(ydb.GenericError))) + + +class TestDocApiTables(WithTenant): + def _create_table(self, path): session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) session.create_table( - path, - ydb.TableDescription() - .with_columns( - ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_primary_key('key') + path, + ydb.TableDescription() + .with_columns( + ydb.Column('key', ydb.OptionalType(ydb.PrimitiveType.Int32)), + ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) + ) + .with_primary_key('key') .with_attributes({'__document_api_version': '1'}) - ) - - def test_create_table(self): - path = '{}/test_doc_api_tables/create_table/table'.format(self.database_name) - self._create_table(path) - - @pytest.mark.parametrize('settings,ex', [ - (None, ydb.BadRequest), - (ydb.BaseRequestSettings().with_request_type('_document_api_request'), None), - ]) - def test_alter_table(self, settings, ex): - path = '{}/test_test_doc_api_tables/alter_table/table'.format(self.database_name) - self._create_table(path) + ) + + def test_create_table(self): + path = '{}/test_doc_api_tables/create_table/table'.format(self.database_name) + self._create_table(path) + + @pytest.mark.parametrize('settings,ex', [ + (None, ydb.BadRequest), + (ydb.BaseRequestSettings().with_request_type('_document_api_request'), None), + ]) + def test_alter_table(self, settings, ex): + path = '{}/test_test_doc_api_tables/alter_table/table'.format(self.database_name) + self._create_table(path) session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) - - def callee(): + + def callee(): session.alter_table( - path, add_columns=[], drop_columns=['value'], settings=settings - ) - - if ex: - assert_that(callee, raises(ex, "Document API table cannot be modified")) - else: - callee() - - @pytest.mark.parametrize('settings,ex', [ - (None, None), - (ydb.BaseRequestSettings().with_request_type('_document_api_request'), None), - ]) - def test_drop_table(self, settings, ex): - path = '{}/test_test_doc_api_tables/drop_table/table'.format(self.database_name) - self._create_table(path) + path, add_columns=[], drop_columns=['value'], settings=settings + ) + + if ex: + assert_that(callee, raises(ex, "Document API table cannot be modified")) + else: + callee() + + @pytest.mark.parametrize('settings,ex', [ + (None, None), + (ydb.BaseRequestSettings().with_request_type('_document_api_request'), None), + ]) + def test_drop_table(self, settings, ex): + path = '{}/test_test_doc_api_tables/drop_table/table'.format(self.database_name) + self._create_table(path) session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create()) - - def callee(): + + def callee(): session.drop_table( - path, settings=settings - ) - - if ex: - assert_that(callee, raises(ex, "Document API table cannot be modified")) - else: - callee() + path, settings=settings + ) + + if ex: + assert_that(callee, raises(ex, "Document API table cannot be modified")) + else: + callee() diff --git a/ydb/tests/functional/limits/test_schemeshard_limits.py b/ydb/tests/functional/limits/test_schemeshard_limits.py index c18a8ce6773..330cb883779 100644 --- a/ydb/tests/functional/limits/test_schemeshard_limits.py +++ b/ydb/tests/functional/limits/test_schemeshard_limits.py @@ -2,7 +2,7 @@ import os import string -from hamcrest import assert_that, raises +from hamcrest import assert_that, raises from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels @@ -42,68 +42,68 @@ class TestSchemeShardLimitsCase0(Base): erasure = Erasure.BLOCK_4_2 def test_effective_acls_are_too_large(self): - # Arrange - directory_name = "/Root/test_effective_acls_are_too_large" - self.driver.scheme_client.make_directory(directory_name) - - # Act - def callee(): - for idx in string.ascii_lowercase: - current_directory = os.path.join(directory_name, idx) - self.driver.scheme_client.make_directory(current_directory) - subject = idx * (6 * 10 ** 6) + '@staff' - self.driver.scheme_client.modify_permissions( - current_directory, - ydb.ModifyPermissionsSettings() - .grant_permissions( - subject, ( - 'ydb.generic.read', - ) + # Arrange + directory_name = "/Root/test_effective_acls_are_too_large" + self.driver.scheme_client.make_directory(directory_name) + + # Act + def callee(): + for idx in string.ascii_lowercase: + current_directory = os.path.join(directory_name, idx) + self.driver.scheme_client.make_directory(current_directory) + subject = idx * (6 * 10 ** 6) + '@staff' + self.driver.scheme_client.modify_permissions( + current_directory, + ydb.ModifyPermissionsSettings() + .grant_permissions( + subject, ( + 'ydb.generic.read', + ) ) ) - self.driver.scheme_client.describe_path(current_directory) - - # Assert - assert_that( - callee, - raises( - ydb.GenericError, - "ACL is too long" + self.driver.scheme_client.describe_path(current_directory) + + # Assert + assert_that( + callee, + raises( + ydb.GenericError, + "ACL is too long" ) - ) + ) class TestSchemeShardLimitsCase1(Base): erasure = Erasure.NONE def test_too_large_acls(self): - # Arrange + # Arrange directory_name = "/Root/test_too_large_acls" - self.driver.scheme_client.make_directory(directory_name) - - # Act - def callee(): - for chr_val in string.ascii_lowercase: - self.driver.scheme_client.modify_permissions( - directory_name, - ydb.ModifyPermissionsSettings() - .grant_permissions( - 'gvit@staff', ( - 'ydb.generic.read', - ) + self.driver.scheme_client.make_directory(directory_name) + + # Act + def callee(): + for chr_val in string.ascii_lowercase: + self.driver.scheme_client.modify_permissions( + directory_name, + ydb.ModifyPermissionsSettings() + .grant_permissions( + 'gvit@staff', ( + 'ydb.generic.read', + ) ) - .grant_permissions( - chr_val * 50 * 10 ** 6 + '@staff', ( - 'ydb.generic.read', - ) + .grant_permissions( + chr_val * 50 * 10 ** 6 + '@staff', ( + 'ydb.generic.read', + ) ) ) - - # Assert - assert_that( - callee, - raises( - ydb.GenericError, - "ACL is too long" + + # Assert + assert_that( + callee, + raises( + ydb.GenericError, + "ACL is too long" ) - ) + ) diff --git a/ydb/tests/functional/scheme_shard/test_scheme_shard_operations.py b/ydb/tests/functional/scheme_shard/test_scheme_shard_operations.py index 4dd33148fce..7c92c832549 100644 --- a/ydb/tests/functional/scheme_shard/test_scheme_shard_operations.py +++ b/ydb/tests/functional/scheme_shard/test_scheme_shard_operations.py @@ -71,23 +71,23 @@ class TestSchemeShardSimpleOps(object): ) ) - def test_create_path_with_long_name_failed(self): + def test_create_path_with_long_name_failed(self): # Act - def callee(): - name = 'test_create_path_with_long_name_success' * 500 - self.ydb_client.scheme_client.make_directory( - os.path.join( - '/', self.root_dir, name, - ) + def callee(): + name = 'test_create_path_with_long_name_success' * 500 + self.ydb_client.scheme_client.make_directory( + os.path.join( + '/', self.root_dir, name, + ) + ) + + # Assert + assert_that( + callee, + raises( + ydb.SchemeError, + "path part is too long" ) - - # Assert - assert_that( - callee, - raises( - ydb.SchemeError, - "path part is too long" - ) ) def test_delete_directory_from_leaf_success(self): diff --git a/ydb/tests/functional/scheme_tests/canondata/result.json b/ydb/tests/functional/scheme_tests/canondata/result.json index 219405b7318..3f7abb99301 100644 --- a/ydb/tests/functional/scheme_tests/canondata/result.json +++ b/ydb/tests/functional/scheme_tests/canondata/result.json @@ -49,4 +49,4 @@ "uri": "file://tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_tx_mediator_/tx_mediator.schema" } } -} +} diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_datashard_/flat_datashard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_datashard_/flat_datashard.schema index bcd702e15fc..10d5f2ef246 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_datashard_/flat_datashard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_datashard_/flat_datashard.schema @@ -1465,72 +1465,72 @@ "Blobs": 1 } } - }, - { - "TableId": 28, - "TableName": "SchemaSnapshots", - "TableKey": [ - 1, - 2, - 3 - ], - "ColumnsAdded": [ - { - "ColumnId": 1, - "ColumnName": "PathOwnerId", - "ColumnType": "Uint64" - }, - { - "ColumnId": 2, - "ColumnName": "LocalPathId", - "ColumnType": "Uint64" - }, - { - "ColumnId": 3, - "ColumnName": "SchemaVersion", - "ColumnType": "Uint64" - }, - { - "ColumnId": 4, - "ColumnName": "Step", - "ColumnType": "Uint64" - }, - { - "ColumnId": 5, - "ColumnName": "TxId", - "ColumnType": "Uint64" - }, - { - "ColumnId": 6, - "ColumnName": "Schema", - "ColumnType": "String" - } - ], - "ColumnsDropped": [], - "ColumnFamilies": { - "0": { - "Columns": [ - 1, - 2, - 3, - 4, - 5, - 6 - ], - "RoomID": 0, - "Codec": 0, - "InMemory": false, - "Cache": 0, - "Small": 4294967295, - "Large": 4294967295 - } - }, - "Rooms": { - "0": { - "Main": 1, - "Outer": 1, - "Blobs": 1 - } - } + }, + { + "TableId": 28, + "TableName": "SchemaSnapshots", + "TableKey": [ + 1, + 2, + 3 + ], + "ColumnsAdded": [ + { + "ColumnId": 1, + "ColumnName": "PathOwnerId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 2, + "ColumnName": "LocalPathId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 3, + "ColumnName": "SchemaVersion", + "ColumnType": "Uint64" + }, + { + "ColumnId": 4, + "ColumnName": "Step", + "ColumnType": "Uint64" + }, + { + "ColumnId": 5, + "ColumnName": "TxId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 6, + "ColumnName": "Schema", + "ColumnType": "String" + } + ], + "ColumnsDropped": [], + "ColumnFamilies": { + "0": { + "Columns": [ + 1, + 2, + 3, + 4, + 5, + 6 + ], + "RoomID": 0, + "Codec": 0, + "InMemory": false, + "Cache": 0, + "Small": 4294967295, + "Large": 4294967295 + } + }, + "Rooms": { + "0": { + "Main": 1, + "Outer": 1, + "Blobs": 1 + } + } } ]
\ No newline at end of file diff --git a/ydb/tests/functional/serverless/test.py b/ydb/tests/functional/serverless/test.py index 92012e75db6..55e6b5d4e2e 100644 --- a/ydb/tests/functional/serverless/test.py +++ b/ydb/tests/functional/serverless/test.py @@ -5,12 +5,12 @@ import time import functools import pytest -from hamcrest import ( - assert_that, - contains_inanyorder, - not_none, -) - +from hamcrest import ( + assert_that, + contains_inanyorder, + not_none, +) + import ydb from tornado import gen @@ -78,8 +78,8 @@ def test_create_table(ydb_hostel_db, ydb_serverless_db, ydb_endpoint, metering_f ) pool.retry_operation_sync(drop_table, None, os.path.join(database, "dirA1", "dirB1", "table")) - - + + def test_turn_on_serverless_storage_billing(ydb_hostel_db, ydb_serverless_db, ydb_endpoint, metering_file_path, ydb_private_client): logger.debug( "test for serverless db %s over hostel db %s", ydb_serverless_db, ydb_hostel_db @@ -412,18 +412,18 @@ def test_database_with_disk_quotas(ydb_hostel_db, ydb_disk_quoted_serverless_db, IOLoop.current().run_sync(lambda: async_write_key(path, 0, 'test', ignore_out_of_space=False)) -def test_discovery(ydb_hostel_db, ydb_serverless_db, ydb_endpoint): - def list_endpoints(database): - logger.debug("List endpoints of %s", database) - resolver = ydb.DiscoveryEndpointsResolver(ydb.DriverConfig(ydb_endpoint, database)) - result = resolver.resolve() - if result is not None: - return result.endpoints - return result - - hostel_db_endpoints = list_endpoints(ydb_hostel_db) - serverless_db_endpoints = list_endpoints(ydb_serverless_db) - - assert_that(hostel_db_endpoints, not_none()) - assert_that(serverless_db_endpoints, not_none()) - assert_that(serverless_db_endpoints, contains_inanyorder(*hostel_db_endpoints)) +def test_discovery(ydb_hostel_db, ydb_serverless_db, ydb_endpoint): + def list_endpoints(database): + logger.debug("List endpoints of %s", database) + resolver = ydb.DiscoveryEndpointsResolver(ydb.DriverConfig(ydb_endpoint, database)) + result = resolver.resolve() + if result is not None: + return result.endpoints + return result + + hostel_db_endpoints = list_endpoints(ydb_hostel_db) + serverless_db_endpoints = list_endpoints(ydb_serverless_db) + + assert_that(hostel_db_endpoints, not_none()) + assert_that(serverless_db_endpoints, not_none()) + assert_that(serverless_db_endpoints, contains_inanyorder(*hostel_db_endpoints)) diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py index 81dcd6e0e1c..05d5303f644 100644 --- a/ydb/tests/functional/sqs/sqs_test_base.py +++ b/ydb/tests/functional/sqs/sqs_test_base.py @@ -273,15 +273,15 @@ class KikimrSqsTestBase(object): @classmethod def _init_cluster(cls, cluster, config_generator): - driver_config = ydb.DriverConfig( - "%s:%s" % (cluster.nodes[1].host, cluster.nodes[1].port), - cls.database - ) - - with ydb.Driver(driver_config) as driver: - driver.wait() - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: + driver_config = ydb.DriverConfig( + "%s:%s" % (cluster.nodes[1].host, cluster.nodes[1].port), + cls.database + ) + + with ydb.Driver(driver_config) as driver: + driver.wait() + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: create_all_sqs_tables(cls.sqs_root, driver, session) cls.create_metauser(cluster, config_generator) diff --git a/ydb/tests/functional/tenants/common.py b/ydb/tests/functional/tenants/common.py index 77f845be96f..827005e5aec 100644 --- a/ydb/tests/functional/tenants/common.py +++ b/ydb/tests/functional/tenants/common.py @@ -120,8 +120,8 @@ class DBForStaticSlots(object): class DBWithDynamicSlot(object): - SLOT_COUNT = 1 - + SLOT_COUNT = 1 + @classmethod def setup_class(cls): cls.cluster = kikimr_cluster_factory( diff --git a/ydb/tests/functional/tenants/test_dynamic_tenants.py b/ydb/tests/functional/tenants/test_dynamic_tenants.py index 7d669c420a4..bdf7d597824 100644 --- a/ydb/tests/functional/tenants/test_dynamic_tenants.py +++ b/ydb/tests/functional/tenants/test_dynamic_tenants.py @@ -3,13 +3,13 @@ import os import logging import time -from hamcrest import ( - any_of, - assert_that, - calling, - equal_to, - raises, -) +from hamcrest import ( + any_of, + assert_that, + calling, + equal_to, + raises, +) import ydb @@ -98,13 +98,13 @@ class TestCreateTenantThenExecYQLEmptyDatabaseHeader(DBWithDynamicSlot): class TestCreateTenantThenExecYQL(DBWithDynamicSlot): def test_case(self): - database = '/Root/users/database' - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - database - ) - + database = '/Root/users/database' + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + database + ) + driver_config2 = ydb.DriverConfig( "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), database + "/" @@ -298,112 +298,112 @@ class TestCreateAndDropTheSameTenant2(DBForStaticSlots): self.cluster.remove_database(database) logger.debug("done %d", iNo) - - -class TestCheckAccess(DBWithDynamicSlot): - SLOT_COUNT = 2 - - def test_case(self): - users = {} - for user in ('user_1', 'user_2'): - users[user] = { - 'path': os.path.join('/Root/users', user), - 'owner': '%s@builtin' % user, - } - - for user in users.values(): - self.cluster.create_database( - user['path'], - storage_pool_units_count={ - 'hdd': 1 - } - ) - + + +class TestCheckAccess(DBWithDynamicSlot): + SLOT_COUNT = 2 + + def test_case(self): + users = {} + for user in ('user_1', 'user_2'): + users[user] = { + 'path': os.path.join('/Root/users', user), + 'owner': '%s@builtin' % user, + } + + for user in users.values(): + self.cluster.create_database( + user['path'], + storage_pool_units_count={ + 'hdd': 1 + } + ) + self.cluster.register_and_start_slots(user['path'], count=1) self.cluster.wait_tenant_up(user['path']) - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - user['path'] - ) - - with ydb.Driver(driver_config) as driver: - driver.wait(timeout=10) - - client = ydb.SchemeClient(driver) - client.modify_permissions( - user['path'], - ydb.ModifyPermissionsSettings().change_owner(user['owner']) - ) - - user_1 = users['user_1'] - user_2 = users['user_2'] - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - user_1['path'], auth_token=user_1['owner'] - ) - - with ydb.Driver(driver_config) as driver: - driver.wait(timeout=10) - client = ydb.SchemeClient(driver) - - while True: - try: - client.list_directory(user_1['path']) - except ydb.Unauthorized: - time.sleep(5) # wait until caches are refreshed - else: - break - - assert_that( - calling(client.list_directory).with_args( - user_2['path'] - ), - raises(ydb.Unauthorized) - ) - - assert_that( - calling(client.list_directory).with_args( - os.path.join(user_1['path'], 'a') - ), - raises(ydb.SchemeError) - ) - assert_that( - calling(client.list_directory).with_args( - os.path.join(user_2['path'], 'a') - ), - raises(ydb.SchemeError) - ) - - client.make_directory(os.path.join(user_1['path'], 'a')) - assert_that( - calling(client.make_directory).with_args( - os.path.join(user_2['path'], 'a') - ), - raises(ydb.BadRequest) - ) - - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - session.execute_scheme( + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + user['path'] + ) + + with ydb.Driver(driver_config) as driver: + driver.wait(timeout=10) + + client = ydb.SchemeClient(driver) + client.modify_permissions( + user['path'], + ydb.ModifyPermissionsSettings().change_owner(user['owner']) + ) + + user_1 = users['user_1'] + user_2 = users['user_2'] + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + user_1['path'], auth_token=user_1['owner'] + ) + + with ydb.Driver(driver_config) as driver: + driver.wait(timeout=10) + client = ydb.SchemeClient(driver) + + while True: + try: + client.list_directory(user_1['path']) + except ydb.Unauthorized: + time.sleep(5) # wait until caches are refreshed + else: + break + + assert_that( + calling(client.list_directory).with_args( + user_2['path'] + ), + raises(ydb.Unauthorized) + ) + + assert_that( + calling(client.list_directory).with_args( + os.path.join(user_1['path'], 'a') + ), + raises(ydb.SchemeError) + ) + assert_that( + calling(client.list_directory).with_args( + os.path.join(user_2['path'], 'a') + ), + raises(ydb.SchemeError) + ) + + client.make_directory(os.path.join(user_1['path'], 'a')) + assert_that( + calling(client.make_directory).with_args( + os.path.join(user_2['path'], 'a') + ), + raises(ydb.BadRequest) + ) + + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + session.execute_scheme( "create table `{}` (id Int64, primary key(id));".format( - os.path.join(user_1['path'], 'q/w/table') - ) - ) - assert_that( - calling(session.execute_scheme).with_args( + os.path.join(user_1['path'], 'q/w/table') + ) + ) + assert_that( + calling(session.execute_scheme).with_args( "create table `{}` (id Int64, primary key(id));".format( - os.path.join(user_2['path'], 'q/w/table') - ) - ), - any_of(raises(ydb.GenericError), raises(ydb.Unauthorized)) - ) - - assert_that( - calling(client.list_directory).with_args( - '/Root/' - ), - raises(ydb.Unauthorized) - ) - client.list_directory('/') + os.path.join(user_2['path'], 'q/w/table') + ) + ), + any_of(raises(ydb.GenericError), raises(ydb.Unauthorized)) + ) + + assert_that( + calling(client.list_directory).with_args( + '/Root/' + ), + raises(ydb.Unauthorized) + ) + client.list_directory('/') diff --git a/ydb/tests/functional/tenants/test_storage_config.py b/ydb/tests/functional/tenants/test_storage_config.py index bc8dfbae64e..94f1789ed58 100644 --- a/ydb/tests/functional/tenants/test_storage_config.py +++ b/ydb/tests/functional/tenants/test_storage_config.py @@ -561,12 +561,12 @@ class TestStorageConfig(TBaseTenant): def test_create_tablet(self): table_path = '%s/table-1' % (self.tenant_path) - with self.pool.checkout() as session: - session.execute_scheme( + with self.pool.checkout() as session: + session.execute_scheme( "create table `{}` (key Int32, value String, primary key(key));".format( - table_path - ) - ) + table_path + ) + ) session.transaction().execute( "upsert into `{}` (key) values (101);".format(table_path), diff --git a/ydb/tests/functional/tenants/test_system_views.py b/ydb/tests/functional/tenants/test_system_views.py index 036e957f383..b0fc5640f62 100644 --- a/ydb/tests/functional/tenants/test_system_views.py +++ b/ydb/tests/functional/tenants/test_system_views.py @@ -1,77 +1,77 @@ -# -*- coding: utf-8 -*- -import logging -import os +# -*- coding: utf-8 -*- +import logging +import os import time - -from hamcrest import ( - anything, - assert_that, + +from hamcrest import ( + anything, + assert_that, greater_than, - has_length, - has_properties, -) - + has_length, + has_properties, +) + from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels import ydb - -logger = logging.getLogger(__name__) - - -class BaseSystemViews(object): - @classmethod - def setup_class(cls): - cls.cluster = kikimr_cluster_factory( - KikimrConfigGenerator( + +logger = logging.getLogger(__name__) + + +class BaseSystemViews(object): + @classmethod + def setup_class(cls): + cls.cluster = kikimr_cluster_factory( + KikimrConfigGenerator( additional_log_configs={ 'SYSTEM_VIEWS': LogLevels.DEBUG } - ) - ) - cls.cluster.start() - - @classmethod - def teardown_class(cls): - if hasattr(cls, 'cluster'): - cls.cluster.stop() - - def setup_method(self, method=None): - self.database = "/Root/users/{class_name}_{method_name}".format( + ) + ) + cls.cluster.start() + + @classmethod + def teardown_class(cls): + if hasattr(cls, 'cluster'): + cls.cluster.stop() + + def setup_method(self, method=None): + self.database = "/Root/users/{class_name}_{method_name}".format( class_name=self.__class__.__name__, method_name=method.__name__, - ) - logger.debug("Create database %s" % self.database) - self.cluster.create_database( - self.database, - storage_pool_units_count={ - 'hdd': 1 - } - ) + ) + logger.debug("Create database %s" % self.database) + self.cluster.create_database( + self.database, + storage_pool_units_count={ + 'hdd': 1 + } + ) self.cluster.register_and_start_slots(self.database, count=1) self.cluster.wait_tenant_up(self.database) - - def teardown_method(self, method=None): - logger.debug("Remove database %s" % self.database) - self.cluster.remove_database(self.database) - self.database = None - - def create_table(self, driver, table): - with ydb.SessionPool(driver, size=1) as pool: - with pool.checkout() as session: - session.execute_scheme( + + def teardown_method(self, method=None): + logger.debug("Remove database %s" % self.database) + self.cluster.remove_database(self.database) + self.database = None + + def create_table(self, driver, table): + with ydb.SessionPool(driver, size=1) as pool: + with pool.checkout() as session: + session.execute_scheme( "create table `{}` (key Int32, value String, primary key(key));".format( - table - ) - ) - - def read_partition_stats(self, driver, database): - table = os.path.join(database, '.sys/partition_stats') + table + ) + ) + + def read_partition_stats(self, driver, database): + table = os.path.join(database, '.sys/partition_stats') return self._stream_query_result( driver, "select PathId, PartIdx, Path from `{}`;".format(table) ) - + def read_query_metrics(self, driver, database): table = os.path.join(database, '.sys/query_metrics_one_minute') return self._stream_query_result( @@ -86,24 +86,24 @@ class BaseSystemViews(object): "select Duration, ReadBytes, CPUTime, RequestUnits from `{}`;".format(table) ) - def _stream_query_result(self, driver, query): + def _stream_query_result(self, driver, query): it = driver.table_client.scan_query(query) - result = [] - - while True: - try: - response = next(it) - except StopIteration: - break - + result = [] + + while True: + try: + response = next(it) + except StopIteration: + break + for row in response.result_set.rows: result.append(row) - return result - + return result + def check_query_metrics_and_stats(self, query_count): table = os.path.join(self.database, 'table') - + driver_config = ydb.DriverConfig( "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), self.database @@ -156,28 +156,28 @@ class BaseSystemViews(object): })) -class TestPartitionStats(BaseSystemViews): - def test_case(self): - for database in ('/Root', self.database): - table = os.path.join(database, 'table') - - driver_config = ydb.DriverConfig( - "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), - database - ) - - with ydb.Driver(driver_config) as driver: - driver.wait(timeout=10) - - self.create_table(driver, table) - stats = self.read_partition_stats(driver, database) - - assert_that(stats, has_length(1)) - assert_that(stats[0], has_properties({ - 'Path': table, - 'PathId': anything(), - 'PartIdx': 0, - })) +class TestPartitionStats(BaseSystemViews): + def test_case(self): + for database in ('/Root', self.database): + table = os.path.join(database, 'table') + + driver_config = ydb.DriverConfig( + "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port), + database + ) + + with ydb.Driver(driver_config) as driver: + driver.wait(timeout=10) + + self.create_table(driver, table) + stats = self.read_partition_stats(driver, database) + + assert_that(stats, has_length(1)) + assert_that(stats[0], has_properties({ + 'Path': table, + 'PathId': anything(), + 'PartIdx': 0, + })) class TestQueryMetrics(BaseSystemViews): diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make index 3474bf0f19a..dee204013e0 100644 --- a/ydb/tests/functional/tenants/ya.make +++ b/ydb/tests/functional/tenants/ya.make @@ -7,7 +7,7 @@ TEST_SRCS( test_dynamic_tenants.py test_tenants.py test_storage_config.py - test_system_views.py + test_system_views.py test_publish_into_schemeboard_with_common_ssring.py ) diff --git a/ydb/tests/functional/ttl/test_ttl.py b/ydb/tests/functional/ttl/test_ttl.py index 793fdaf2b34..a822a2d61fb 100644 --- a/ydb/tests/functional/ttl/test_ttl.py +++ b/ydb/tests/functional/ttl/test_ttl.py @@ -1,182 +1,182 @@ -# -*- coding: utf-8 -*- - -import os -import time - -from hamcrest import ( - any_of, - assert_that, - has_item, - has_length, - has_properties, - none, - not_none, -) - +# -*- coding: utf-8 -*- + +import os +import time + +from hamcrest import ( + any_of, + assert_that, + has_item, + has_length, + has_properties, + none, + not_none, +) + from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels import ydb - - -class TestTTL(object): - ENABLE_TTL_ON_INDEXED_TABLES = None - - @classmethod - def setup_class(cls): - cls.cluster = kikimr_cluster_factory(KikimrConfigGenerator( - additional_log_configs={ - 'FLAT_TX_SCHEMESHARD': LogLevels.DEBUG, - 'TX_DATASHARD': LogLevels.DEBUG, - }, - )) - cls.cluster.start() - cls.endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port) - cls.database = '/Root' - - @classmethod - def teardown_class(cls): - cls.cluster.stop() - - @classmethod - def build_table_description(cls): - return ydb.TableDescription().with_primary_keys( - 'id' - ).with_columns( - ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Timestamp)), - ).with_ttl( - ydb.TtlSettings().with_date_type_column('expire_at') - ) - - @classmethod - def upsert(cls, session, table): - session.transaction().execute( - 'upsert into `{}` (id, expire_at) values' - '(1, cast("1970-01-01T00:00:00.000000Z" as Timestamp)),' - '(2, cast("1990-03-01T00:00:00.000000Z" as Timestamp)),' - '(3, cast("2030-04-15T00:00:00.000000Z" as Timestamp));'.format(table), - commit_tx=True - ) - - def _run_test(self): - with ydb.Driver(ydb.DriverConfig(self.endpoint, self.database)) as driver: - with ydb.SessionPool(driver) as pool: - with pool.checkout() as session: - table = os.path.join(self.database, 'table_with_ttl_column') - - session.create_table(table, self.build_table_description()) - - description = session.describe_table(table) - assert_that(description.ttl_settings, not_none()) - assert_that(any_of( - description.ttl_settings.date_type_column, - description.ttl_settings.value_since_unix_epoch - ), not_none()) - - self.upsert(session, table) - + + +class TestTTL(object): + ENABLE_TTL_ON_INDEXED_TABLES = None + + @classmethod + def setup_class(cls): + cls.cluster = kikimr_cluster_factory(KikimrConfigGenerator( + additional_log_configs={ + 'FLAT_TX_SCHEMESHARD': LogLevels.DEBUG, + 'TX_DATASHARD': LogLevels.DEBUG, + }, + )) + cls.cluster.start() + cls.endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port) + cls.database = '/Root' + + @classmethod + def teardown_class(cls): + cls.cluster.stop() + + @classmethod + def build_table_description(cls): + return ydb.TableDescription().with_primary_keys( + 'id' + ).with_columns( + ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), + ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Timestamp)), + ).with_ttl( + ydb.TtlSettings().with_date_type_column('expire_at') + ) + + @classmethod + def upsert(cls, session, table): + session.transaction().execute( + 'upsert into `{}` (id, expire_at) values' + '(1, cast("1970-01-01T00:00:00.000000Z" as Timestamp)),' + '(2, cast("1990-03-01T00:00:00.000000Z" as Timestamp)),' + '(3, cast("2030-04-15T00:00:00.000000Z" as Timestamp));'.format(table), + commit_tx=True + ) + + def _run_test(self): + with ydb.Driver(ydb.DriverConfig(self.endpoint, self.database)) as driver: + with ydb.SessionPool(driver) as pool: + with pool.checkout() as session: + table = os.path.join(self.database, 'table_with_ttl_column') + + session.create_table(table, self.build_table_description()) + + description = session.describe_table(table) + assert_that(description.ttl_settings, not_none()) + assert_that(any_of( + description.ttl_settings.date_type_column, + description.ttl_settings.value_since_unix_epoch + ), not_none()) + + self.upsert(session, table) + # conditional erase runs every 60 second for i in range(60): time.sleep(4) - + content = list(self._read_table(session, table, columns=('id',))) - if len(content) == 1: - break - - content = list(self._read_table(session, table, columns=('id',))) - assert_that(content, has_length(1)) - assert_that(content, has_item(has_properties(id=3))) - - @staticmethod - def _read_table(session, *args, **kwargs): - for chunk in iter(session.read_table(*args, **kwargs)): - for row in chunk.rows: - yield row - - + if len(content) == 1: + break + + content = list(self._read_table(session, table, columns=('id',))) + assert_that(content, has_length(1)) + assert_that(content, has_item(has_properties(id=3))) + + @staticmethod + def _read_table(session, *args, **kwargs): + for chunk in iter(session.read_table(*args, **kwargs)): + for row in chunk.rows: + yield row + + class TestTTLDefaultEnv(TestTTL): - def test_case(self): - self._run_test() - - -class TestTTLOnIndexedTable(TestTTL): - ENABLE_TTL_ON_INDEXED_TABLES = True - - @classmethod - def build_table_description(cls): - return super(TestTTLOnIndexedTable, cls).build_table_description().with_index( - ydb.TableIndex('by_expire_at').with_index_columns('expire_at') - ) - - def test_case(self): - self._run_test() - - -class TestTTLValueSinceUnixEpoch(TestTTL): - @classmethod - def build_table_description(cls): - return ydb.TableDescription().with_primary_keys( - 'id' - ).with_columns( - ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ).with_ttl( - ydb.TtlSettings().with_value_since_unix_epoch('expire_at', ydb.ColumnUnit.UNIT_SECONDS) - ) - - @classmethod - def upsert(cls, session, table): - session.transaction().execute( - 'upsert into `{}` (id, expire_at) values' - '(1, 0),' - '(2, 636249600),' - '(3, 1902441600);'.format(table), - commit_tx=True - ) - - def test_case(self): - self._run_test() - - -class TestTTLAlterSettings(object): - @classmethod - def setup_class(cls): - cls.cluster = kikimr_cluster_factory(KikimrConfigGenerator()) - cls.cluster.start() - cls.endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port) - cls.database = '/Root' - - @classmethod - def teardown_class(cls): - cls.cluster.stop() - - def test_case(self): - with ydb.Driver(ydb.DriverConfig(self.endpoint, self.database)) as driver: - with ydb.SessionPool(driver) as pool: - with pool.checkout() as session: - table = os.path.join(self.database, 'table_with_ttl_column') - - session.create_table( - table, ydb.TableDescription() - .with_primary_keys('id') - .with_columns( - ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Timestamp)), - ) - ) - - description = session.describe_table(table) - assert_that(description.ttl_settings, none()) - - session.alter_table(table, set_ttl_settings=ydb.TtlSettings().with_date_type_column('expire_at')) - - description = session.describe_table(table) - assert_that(description.ttl_settings, not_none()) - assert_that(description.ttl_settings.date_type_column, not_none()) - assert_that(description.ttl_settings.date_type_column, has_properties(column_name='expire_at')) - - session.alter_table(table, drop_ttl_settings=True) - - description = session.describe_table(table) - assert_that(description.ttl_settings, none()) + def test_case(self): + self._run_test() + + +class TestTTLOnIndexedTable(TestTTL): + ENABLE_TTL_ON_INDEXED_TABLES = True + + @classmethod + def build_table_description(cls): + return super(TestTTLOnIndexedTable, cls).build_table_description().with_index( + ydb.TableIndex('by_expire_at').with_index_columns('expire_at') + ) + + def test_case(self): + self._run_test() + + +class TestTTLValueSinceUnixEpoch(TestTTL): + @classmethod + def build_table_description(cls): + return ydb.TableDescription().with_primary_keys( + 'id' + ).with_columns( + ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), + ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Uint64)), + ).with_ttl( + ydb.TtlSettings().with_value_since_unix_epoch('expire_at', ydb.ColumnUnit.UNIT_SECONDS) + ) + + @classmethod + def upsert(cls, session, table): + session.transaction().execute( + 'upsert into `{}` (id, expire_at) values' + '(1, 0),' + '(2, 636249600),' + '(3, 1902441600);'.format(table), + commit_tx=True + ) + + def test_case(self): + self._run_test() + + +class TestTTLAlterSettings(object): + @classmethod + def setup_class(cls): + cls.cluster = kikimr_cluster_factory(KikimrConfigGenerator()) + cls.cluster.start() + cls.endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port) + cls.database = '/Root' + + @classmethod + def teardown_class(cls): + cls.cluster.stop() + + def test_case(self): + with ydb.Driver(ydb.DriverConfig(self.endpoint, self.database)) as driver: + with ydb.SessionPool(driver) as pool: + with pool.checkout() as session: + table = os.path.join(self.database, 'table_with_ttl_column') + + session.create_table( + table, ydb.TableDescription() + .with_primary_keys('id') + .with_columns( + ydb.Column('id', ydb.OptionalType(ydb.PrimitiveType.Uint64)), + ydb.Column('expire_at', ydb.OptionalType(ydb.PrimitiveType.Timestamp)), + ) + ) + + description = session.describe_table(table) + assert_that(description.ttl_settings, none()) + + session.alter_table(table, set_ttl_settings=ydb.TtlSettings().with_date_type_column('expire_at')) + + description = session.describe_table(table) + assert_that(description.ttl_settings, not_none()) + assert_that(description.ttl_settings.date_type_column, not_none()) + assert_that(description.ttl_settings.date_type_column, has_properties(column_name='expire_at')) + + session.alter_table(table, drop_ttl_settings=True) + + description = session.describe_table(table) + assert_that(description.ttl_settings, none()) diff --git a/ydb/tests/functional/ttl/ya.make b/ydb/tests/functional/ttl/ya.make index 78454c829e0..dd1ad15ead9 100644 --- a/ydb/tests/functional/ttl/ya.make +++ b/ydb/tests/functional/ttl/ya.make @@ -1,29 +1,29 @@ PY3TEST() - -OWNER( - ilnaz - g:kikimr -) - -TEST_SRCS( - test_ttl.py -) - + +OWNER( + ilnaz + g:kikimr +) + +TEST_SRCS( + test_ttl.py +) + ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") -TIMEOUT(600) -SIZE(MEDIUM) - -DEPENDS( +TIMEOUT(600) +SIZE(MEDIUM) + +DEPENDS( ydb/apps/ydbd -) - -PEERDIR( +) + +PEERDIR( ydb/tests/library ydb/public/sdk/python/ydb - contrib/python/PyHamcrest -) - -FORK_SUBTESTS() -FORK_TEST_FILES() - -END() + contrib/python/PyHamcrest +) + +FORK_SUBTESTS() +FORK_TEST_FILES() + +END() diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index c45aa1e1167..251f485b5db 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -253,13 +253,13 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): logger.debug("Executing command = {}".format(full_command)) try: return yatest_common.execute(full_command) - except yatest_common.ExecutionError as e: - logger.exception("KiKiMR command '{cmd}' failed with error: {e}\n\tstdout: {out}\n\tstderr: {err}".format( - cmd=" ".join(str(x) for x in full_command), - e=str(e), - out=e.execution_result.std_out, - err=e.execution_result.std_err - )) + except yatest_common.ExecutionError as e: + logger.exception("KiKiMR command '{cmd}' failed with error: {e}\n\tstdout: {out}\n\tstderr: {err}".format( + cmd=" ".join(str(x) for x in full_command), + e=str(e), + out=e.execution_result.std_out, + err=e.execution_result.std_err + )) raise def start(self): @@ -456,9 +456,9 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): self._bs_config_invoke(request) def _bs_config_invoke(self, request): - timeout = yatest_common.plain_or_under_sanitizer(120, 240) - sleep = 5 - retries, success = timeout / sleep, False + timeout = yatest_common.plain_or_under_sanitizer(120, 240) + sleep = 5 + retries, success = timeout / sleep, False while retries > 0 and not success: try: self.__call_kikimr_new_cli( @@ -475,7 +475,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): except Exception as e: logger.error("Failed to execute, %s", str(e)) retries -= 1 - time.sleep(sleep) + time.sleep(sleep) if retries == 0: raise |